From fe7219d186bcb3522756e687cf52dbb41e87650a Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 16 Jun 2026 18:03:32 -0700 Subject: [PATCH 01/66] docs(concurrent-keys): add Phase 1 design and plan Canonical reference docs for the concurrent SDK keys project, covering: - phase1-design.md: architecture, model (anchor, accepted set, expiry, events), wire format, decisions and rationale, open questions. - phase1-plan.md: branching strategy, wave breakdown, task list with dependencies and estimates, test strategy, rollout, JIRA structure. These docs are the source of truth for the work tracked under SDK-2453. Sub-task tickets reference specific sections for design rationale. --- docs/concurrent-keys/phase1-design.md | 407 +++++++++++++++++++++++++ docs/concurrent-keys/phase1-plan.md | 416 ++++++++++++++++++++++++++ 2 files changed, 823 insertions(+) create mode 100644 docs/concurrent-keys/phase1-design.md create mode 100644 docs/concurrent-keys/phase1-plan.md diff --git a/docs/concurrent-keys/phase1-design.md b/docs/concurrent-keys/phase1-design.md new file mode 100644 index 00000000..0b46e4c5 --- /dev/null +++ b/docs/concurrent-keys/phase1-design.md @@ -0,0 +1,407 @@ +# Phase 1 — Concurrent SDK Keys in Relay Proxy: Design + +**Epic**: [SDK-2453](https://launchdarkly.atlassian.net/browse/SDK-2453) +**Backend tech spec**: [Confluence 4186243250](https://launchdarkly.atlassian.net/wiki/spaces/PD/pages/4186243250/Tech+Spec+Concurrent+SDK+Keys) +**Companion**: [`phase1-plan.md`](./phase1-plan.md) (tasks, sequencing, estimates) + +This document is the canonical reference for the *what* and *why* of Phase 1. The companion plan covers *how*. Agents working on individual tasks should read both. + +--- + +## 1. Overview + +LaunchDarkly is rolling out **concurrent SDK keys** — the ability for a single environment to have *multiple* SDK keys and *multiple* mobile keys simultaneously. Phase 1 brings this capability to the Relay Proxy. + +### Why + +Customers (notably Block/Square, Confluent) maintain dozens or hundreds of services that share the same LaunchDarkly environment. Today each service uses the same single SDK key per environment. If that key is compromised, customers face large-scale operational toil rotating it across every service. + +Concurrent keys let customers issue distinct keys per service, reducing blast radius and supporting independent key lifecycles. + +### Scope + +| In scope (Phase 1) | Out of scope | +|---|---| +| Multiple SDK keys per environment | Multiple client-side IDs per environment (deferred to a later project) | +| Multiple mobile keys per environment | Views / payload filtering V2 (Phase 2 — mega stream) | +| Per-key expiry and graceful rotation | Per-key event attribution (analytics events collapse to the env's anchor key) | +| Delivery via **Relay Auto Config (RAC)** | Manual config (TOML / env vars) — stays single-key in Phase 1 (lifted in Phase 2) | +| Delivery via **offline-mode archive** | | +| Implementation in Relay v8, merged forward to Relay v9 | | + +### The "trusted source" restriction + +Relay authenticates *upstream* with only one SDK key per environment — the **anchor**. Additional keys are accepted locally but not verified upstream. That makes "trust the source of additional keys" load-bearing for safety. + +LaunchDarkly-generated sources (RAC, offline archive) are trusted: they only ever carry an environment's real keys, so a wrong-environment key can't appear. Hand-entered manual config is *not* trusted — a typo could silently leak this env's data to an SDK using a wrong-env key and misattribute its events to this env. Phase 1 therefore accepts additional keys *only* from RAC and offline archives. + +Manual config support returns in Phase 2 via the mega stream, which verifies every key individually. + +Customer impact: ~50% of relay customers use RAC and benefit immediately. The other ~50% (manual config) wait for Phase 2. The team has reviewed this trade-off — see [Confluence 4979425298](https://launchdarkly.atlassian.net/wiki/spaces/PD/pages/4979425298/Relay+Proxy+Auto+Config+Risk+Assessment). + +--- + +## 2. Preamble: How Relay Auto Config (RAC) works + +The §2.4 event table is unreadable without this context. + +**RAC is a push channel from LaunchDarkly *to* relay.** It's a single long-lived SSE stream over HTTPS. Relay opens it at startup using a special "relay token" (distinct from any SDK key) and consumes the stream's messages. + +**Lifecycle**: +1. Relay starts up. If RAC is configured, relay opens an SSE connection to LaunchDarkly's RAC endpoint. +2. LaunchDarkly responds with an initial `put` message containing the full state of every environment this relay should know about. Relay creates an `EnvContext` per env and opens upstream SDK clients on the anchor key. +3. While the SSE stream is open, LaunchDarkly pushes incremental messages: `patch /environments/$ENVID` (state changed), `delete /environments/$ENVID` (removed), `put /` (full refresh — rare, usually on reconnect). +4. Connection drops → relay reconnects with backoff and reconciles against the next `put`. + +The path notation (`/environments/$ENVID`) is a **JSON path within the RAC document**, not an HTTP route. It identifies which part of relay's internal state the message addresses. Think JSON Patch semantics. + +**Offline mode is the same in shape, different in transport.** No stream. LaunchDarkly tooling generates an archive file with the same `EnvironmentRep` shape. Relay reads it on startup and reconciles on reload. + +**Don't confuse RAC with the SDK streaming endpoint.** Relay has two kinds of upstream connection: RAC (one per relay, carries config) and the SDK stream (one per env, anchored by SDK key, carries flag/segment data). Phase 1 changes the SDK-stream-per-env story; RAC itself is unchanged in transport. + +## 3. Preamble: RAC vs Manual Config + +Relays use *either* manual config *or* RAC — not both. They're alternative top-level configuration approaches. + +- **Manual config**: TOML file or `LD_*` env vars list each environment explicitly with its SDK key, mobile key, and env ID. Static — operator edits the file and reloads. +- **RAC**: TOML file has a single `[AutoConfig]` block with a relay token. LaunchDarkly streams the environment list. No `[Environment ...]` blocks needed. + +A relay instance is one or the other, decided at deployment time. + +RAC is **enterprise-only** in LaunchDarkly's pricing tiers. Lower-tier customers physically cannot use RAC and use manual config by necessity. This pricing reality is why the trusted-source restriction (§1) excludes ~50% of relay customers from Phase 1. + +--- + +## 4. Architectural pillars + +Phase 1 rests on three commitments. The whole design is consistent with these; they hold across every code path and test. + +### 4.1 One upstream connection per environment, on the anchor + +Relay opens exactly one upstream SDK client per env, authenticated with the anchor SDK key. All other accepted keys (server and mobile) are matched locally against the request's `Authorization` header and served off the same data store. They never open their own upstream connection. + +This generalizes existing behavior: today's mobile keys and client-side IDs already behave this way (verified in [`internal/relayenv/env_context_impl.go`](../../internal/relayenv/env_context_impl.go); only `config.SDKKey` calls `startSDKClient`). Phase 1 extends "local match only, no upstream client" to non-anchor *server* keys. + +**Trade-off**: this is a deliberate choice over the alternative (one upstream client per accepted key, the approach in Matthew Keeler's PoC at PR #675). Reasons we chose single-anchor: +- **Connection-count efficiency** — a customer with 50 keys × 10 envs × 10 filters would otherwise hold thousands of upstream streams. +- **Phase 2 alignment** — Phase 2's mega stream is one connection per environment. Single-anchor is closer to that target. + +The cost is re-anchoring complexity (§7). We accept that cost. + +### 4.2 The anchor + +The anchor is **the SDK key the singular `sdkKey.value` field points to**, identified by byte-equality against an entry in the `sdkKeys[]` array. There is no `isDefault` flag in the wire format — the value match *is* the signal. + +The backend designates the anchor; relay is passive. Relay reads `sdkKey.value` and uses that key for upstream. + +**Invariants** (maintained by the backend): +- `sdkKey` always names a non-expiring key. +- The backend blocks deleting or expiring the last non-expiring key in an environment. On default rotation, the backend promotes another non-expiring key first, then flips `sdkKey.value`. +- The new anchor's entry in `sdkKeys[]` continues to carry no `expiry`. The old anchor (now demoted) carries an `expiry`. + +**Re-anchor trigger**: whenever `sdkKey.value` changes. This is the single trigger for an upstream-client swap. See §7 for the mechanism. + +**Mobile-key analog**: the singular `mobKey` field is the default mobile key for events. No upstream connection — mobile keys are local-match-only — but `mobKey` plays the same back-compat singular-pointer role. + +### 4.3 Events collapse to anchor per kind + +Analytics events forward upstream under the env's anchor key of each kind. Two dispatchers per env: one for SDK events under `sdkKey.value`, one for mobile events under `mobKey`. The dispatcher uses its stored `authKey`, not the credential on the incoming request. + +**Why no per-key event attribution**: SDK keys are *secrets* and not appropriate as metric/analytics tags. LaunchDarkly provides customer-facing tagging mechanisms (context attributes, environment tags) for slicing events. The trusted-source restriction makes anchor attribution safe — every accepted key truly belongs to this env, so anchor attribution lands on the right env. We lose per-key granularity, not env correctness. + +**Asymmetry — diagnostic events**: diagnostic events (SDK self-reported initialization, errors) take a different code path. They proxy the incoming request's headers verbatim, including the Authorization header carrying the original credential. This preserves operational debug value — *which* SDK reported this — at the cost of asymmetry with analytics. We accept this. (Long-term, a metadata-header approach could provide symmetric attribution; out of scope for Phase 1.) + +--- + +## 5. Wire format + +Both RAC and the offline archive carry the same `EnvironmentRep`. One parsing change covers both sources. Producers already emit this format — relay can implement and test against captured payloads today. + +### Example RAC `event:put` + +```json +{ + "path": "/", + "data": { + "environments": { + "68e5179e8307e4099c277e2a": { + "envId": "68e5179e8307e4099c277e2a", + "envKey": "production", + "envName": "Production", + "projKey": "...", + "projName": "...", + "secureMode": false, + "version": 26, + "sdkKey": { "value": "sdk-9409..." }, + "mobKey": "mob-f41c...", + "sdkKeys": [ + { "key": "new-production-default", "value": "sdk-9409..." }, + { "key": "another-one", "value": "sdk-38b0..." } + ], + "mobileKeys": [ + { "key": "mob-key-50bca22351", "value": "mob-f41c..." } + ] + } + } + } +} +``` + +The offline archive wraps the same `env` object per entry: `{"env": ..., "dataId": "..."}`. + +### Shape rules + +- Array entries: `{ "key": , "value": , "expiry"?: }`. +- Singular `sdkKey` is an **object** (`{"value": ...}`); may carry the legacy `expiring{value, timestamp}` slot during default rotation. +- Singular `mobKey` is a **plain string**. Shape asymmetry is historical (mobile keys never had a legacy expiring slot). +- Anchor = `sdkKeys[]` entry whose `value` matches `sdkKey.value`. No `isDefault` flag — value match is the signal. +- Arrays are *inclusive* of the default — the anchor entry is *in* `sdkKeys[]`, not separate. +- `expiry` is present only while a key is expiring; omitted otherwise; never null. +- The legacy `sdkKey.expiring{}` slot is populated **only during default rotation** (old-relay back-compat). Non-default key expiring uses only the array `expiry`. +- Old relays ignore unknown JSON fields and continue using singular `sdkKey`/`mobKey` — additive, fully backward-compatible. + +### Terminology + +Aligned with the backend tech spec's `accounts.sdk_keys` table: + +- **`name`** = display name (e.g. "Default SDK Key"). Used in the UI. **Not in the wire format** — relay doesn't need it. +- **`key`** = identifier (e.g. "default-sdk"). Non-secret. Carried in wire as `key`. +- **`value`** = the credential secret (e.g. `sdk-xxxx-...`). Carried in wire as `value`. + +**Naming trap in code**: relay's existing types `SDKKey`, `MobileKey`, `SDKCredential` refer to what the wire format calls `value`. The wire's `key` field is the *identifier*, a different thing. Do not rename the existing relay types — they're stable — but call out the trap in code comments. + +A canonical comment for the wire-type definition (subject to bikeshed at PR time): + +```go +// EnvironmentRep carries an environment's wire shape from RAC and the offline +// archive (same struct serves both — keep them aligned). +// +// FIELD NAMING — read this before changing anything: +// +// sdkKey is the singular *default* SDK key for the environment. It's an +// object ({"value": "sdk-..."}) so it can also carry the legacy +// sdkKey.expiring{value, timestamp} slot during default rotation +// (back-compat for relays predating concurrent keys). +// +// mobKey is the singular default mobile key. It's a *plain string* +// because mobile keys never had a legacy expiring slot. The shape +// asymmetry is historical, not a design choice. +// +// sdkKeys/mobileKeys are the authoritative full accepted set. Entries: +// { key: , value: , expiry?: } +// +// TERMINOLOGY: +// The wire "key" field is the human-readable IDENTIFIER (e.g. "default-sdk"), +// non-secret. The wire "value" field is the actual CREDENTIAL string (e.g. +// "sdk-xxxx-..."), which is the secret. Note that relay's own types +// (SDKKey, MobileKey, SDKCredential) refer to what the wire calls "value" — +// they're misnamed by today's standards but stable, so do not rename. +// +// Anchor selection: anchor = the sdkKeys entry whose `value` matches +// sdkKey.value. No isDefault flag. See phase1-design.md §4.2. +``` + +--- + +## 6. Credential lifecycle + +### 6.1 Expiry model + +Each entry in `sdkKeys[]` / `mobileKeys[]` carries an optional `expiry` field (Unix-ms timestamp). When present, the key is being phased out — relay drops it when `expiry` passes. When absent, the key is permanent. + +**Two removal paths**: + +- **Graceful**: key has `expiry` set. Relay's existing periodic ticker (`StepTime` → `cleanupExpiredCredentials`) drops the key when the timestamp passes and disconnects downstream SDKs using it. +- **Immediate**: key omitted from the next RAC patch / archive reload. Relay diffs the accepted set on reconcile, finds the missing key, and revokes it now. + +**Edge case**: a key was in graceful state, then omitted entirely → treat as immediate (race-ahead-of-timer). + +### 6.2 Generalize the `Rotator` + +Today's `Rotator` ([`internal/credential/rotator.go`](../../internal/credential/rotator.go)) tracks one primary SDK key + one deprecated-with-expiry slot + single primary mobile key + single primary env ID. Generalize to: a *set* of accepted keys (server + mobile) with optional per-key expiry, plus a designated anchor. + +**Reuse the existing `StepTime` machinery** — generalize from the single `expiring` slot to per-array-key. No new periodic infrastructure. + +**Mobile-key panic**: today, `Rotator.RotateWithGrace(MobileKey, gracePeriod)` panics with `"programmer error: mobile keys do not support deprecation"`. The panic is a guard against an unsupported API state, not a safeguard against a hazard — there was no data-model slot for an expiring mobile key, so the code failed loud rather than store junk. Phase 1's data-model generalization provides the slot; the panic guard is removed alongside. + +### 6.3 Legacy `sdkKey.expiring{}` back-compat + +On default rotation the backend mirrors expiry info into both: +- The old default's entry in `sdkKeys[]` gets `expiry: ` (new field). +- The legacy `sdkKey.expiring{value, timestamp}` slot gets the same (old field, for old relays). + +**Decision**: new relays trust the array. The legacy `sdkKey.expiring{}` field is treated as a write-only back-compat shim — new relays do not read it. (Working assumption pending team confirmation.) + +--- + +## 7. Re-anchoring + +When `sdkKey.value` changes (voluntary rotation *or* current default expiring and being replaced by a promoted non-expiring key), relay must swap its upstream client to the new anchor while preserving downstream SDK connections. + +This is the highest-risk piece of Phase 1. The source plan describes the swap in a single sentence ("stand up the new client, hand the data source over, retire the old"). The actual mechanism is *mostly implicit* in today's code rather than orchestrated: + +- The data store is shared via `storeAdapter`, so two SDK clients pointed at the same env can feed the same store as a side-effect. +- `GetClient()` returns `c.clients[c.keyRotator.SDKKey()]` — so flipping the rotator's anchor swaps which client serves. +- Downstream lookup is on `ScopedCredential`, not on the anchor — so downstream connections route correctly regardless of which anchor is current. + +But several components are wired at **construction time** to the original SDK key and are *not* re-wired by the implicit handoff: + +| Component | Today's wiring | Re-anchor story | +|---|---|---| +| Event dispatcher | Stores `authKey`; has `ReplaceCredential` | Call `ReplaceCredential` on re-anchor | +| Metrics publisher | Stores `authKey`; has `ReplaceCredential` | Call `ReplaceCredential` on re-anchor | +| Big-segment sync | Wired to SDK key at construction | **No re-wire path today — new mechanism needed** | +| `httpconfig` | Built with SDK key at construction | Likely key-independent (TLS / proxy config); verify in PoC | + +### PoC first + +The re-anchor mechanism is the topic of **T0** — a PoC that validates the swap with concrete tests *before* T2 implements it. The PoC answers seven hypotheses: + +1. Two clients sharing a store don't corrupt store invariants. +2. Downstream SSE connections tolerate the swap. +3. Big-segment sync keeps working after re-anchor — or, if not, what re-wiring is needed. +4. `httpconfig` stays functional after re-anchor. +5. Order of operations: start-new → swap pointer → close-old vs. alternatives. +6. Behavior during the swap window (requests arriving mid-swap). +7. Failure modes: new client init fails — recovery behavior. + +T0's deliverable is durable test code that survives into T2. + +--- + +## 8. Processing & lifecycle + +Three event types, two source paths (RAC + offline), one integration point (`ReconcileCredentials` on `EnvContext`). + +| Event | RAC trigger | Offline trigger | Relay action | +|---|---|---|---| +| Env added | `patch /environments/$ENVID` (env not known) | new archive entry on reload | Create env; open upstream connection on anchor (online only); map all accepted keys into lookup | +| Keys change | `patch /environments/$ENVID` (env known, payload differs) | archive update on reload | Reconcile accepted `sdkKeys`/`mobileKeys`; re-anchor if `sdkKey.value` changed | +| Env deleted | `delete /environments/$ENVID` | env removed from reloaded archive | Tear down env + upstream connection + mappings | + +### Order of operations (keys change) + +Within a single `keys change` event, the order is: + +1. **Add new keys** (new credential entries added to accepted set, handlers built). +2. **Re-anchor** if `sdkKey.value` changed (swap upstream client, re-wire peripheral components). +3. **Remove expiring/omitted keys** (drop entries, disconnect downstream SDKs that were using them). + +This order ensures the accepted set is a *superset* during the transition. The new anchor's client comes up before the old anchor's client tears down. Downstream SDKs are never spuriously rejected mid-update. + +### Atomicity + +Reconcile is **all-or-nothing**. On partial failure (malformed payload, new-client init failure, etc.), log a structured error and preserve the previous accepted set. Working assumption — open question for the team. Aligns with the malformed-payload policy (§9). + +### Edge cases + +- **De-expiry** (key was expiring; new payload omits `expiry`): cancel the scheduled drop. +- **Rename** (same `value`, different `key` identifier): no-op for credential set; only update status-endpoint display. +- **Mixed update** (add + re-anchor + remove in one patch): apply in the order above. + +--- + +## 9. Defensive behavior — malformed payloads + +When relay receives a malformed RAC payload — most importantly, `sdkKey.value` not present in `sdkKeys[]`, or `sdkKey` field missing entirely — the backend invariants of §4.2 have been violated. + +**Working assumption** (pending team confirmation): + +- **Hard-fail the update.** Log a structured error. Preserve the previous accepted set. Alarm. +- Do *not* silently fall back to the first entry in `sdkKeys[]` (silent and dangerous). +- Do *not* leave the env in a half-applied state. + +This is the same atomicity principle as §8, applied at the boundary between trusted-source input and relay's internal state. + +--- + +## 10. Backwards compatibility + +Two assertions: + +1. **Payload is additive.** Relays that don't understand the new `sdkKeys`/`mobileKeys` array fields ignore them and continue using the singular `sdkKey`/`mobKey`. No coordinated upgrade required. +2. **One representation for all relays and both sources.** No per-version fork of `EnvironmentRep`. + +**Bidirectional upgrade compat**: customers can upgrade their backend before relay, or relay before backend, in any order. The slowest party uses singular fields; the faster party emits arrays. Both states converge to single-key behavior until both are upgraded. + +**Verification**: `DisallowUnknownFields` is not used anywhere in relay's env-parse path. The additive claim holds — Go's default JSON decoder silently ignores unknown fields. Confirmed as T3.a pre-work. + +**Downgrade story**: open question for the team. Rolling relay back from Phase 1 to a pre-Phase-1 build means SDKs using non-anchor keys would lose connectivity. Document as a release-note consideration. + +--- + +## 11. Manual configuration + +Manual config (TOML file or `LD_*` env vars) continues to support **exactly one SDK key + one mobile key + one env ID per environment**, as today. The schema doesn't change. Manual-config customers see zero behavior change from Phase 1. + +**The PoC's manual-multi-key additions must not be inherited.** SDK-2415 added `AdditionalSDKKeys` and `LD_ADDITIONAL_SDK_KEYS_*` support. We deliberately *do not* want this. T3 review must verify neither pattern appears in `config/config.go` or `config/config_validation.go`. + +--- + +## 12. Internal model + +``` +Environment + envID + identifiers (key, name, proj…) + anchorKey (the one upstream-auth key) + acceptedKeys: KeySet (server + mobile, local match) + clientSideID (single) + upstreamConnection (one, on the anchor) + dataStore + │ + └─── KeySet + keys (equivalent peers, server or mobile) + per-key optional expiry +``` + +**`KeySet`** generalizes today's `Rotator` ("primary + deprecated-with-expiry") into "set of accepted keys + anchor." + +**Routing/auth is a local lookup**: a connecting credential is matched against the accepted set → the environment → served off the single anchor connection. The env ID registers exactly once. + +**No per-view structure anywhere in Phase 1.** Premature abstraction — Phase 2's mega stream is still speculative. Keep the model flat. A key is just an accepted credential. Don't preemptively add `viewKeys` fields or "scope" abstractions. + +--- + +## 13. Recorded decisions + +| Decision | Rationale | Alternatives rejected | +|---|---|---| +| Trusted sources only (RAC + offline archive) in Phase 1 | Relay can't verify additional keys upstream; trusted sources guarantee correct env→key mapping | Manual multi-key with verification (no suitable verification endpoint; staleness problem; Phase 2 resolves anyway), opt-in unsafe flag (same concerns) | +| Single upstream connection per env on the anchor | Connection-count efficiency at scale; aligns with Phase 2's single-mega-stream model | Multi-client (SDK-2415 PoC approach): trades re-anchor complexity for fan-out at customer scale | +| Anchor by `sdkKey.value` byte-match (no `isDefault` flag) | Single source of truth; matches what RAC already emits | `isDefault` flag (would require backend wire change and dual sources of truth) | +| Per-key `expiry` (Unix-ms) on array entries | Confirmed real format from producers; reuses existing ticker | Per-env single deprecated slot (today's model — doesn't scale to multi-key) | +| Trust the array on expiry disagreement (Q7 working assumption) | Simpler invariant; legacy field becomes write-only shim | Take whichever is later, hard-fail on disagreement (more complex, no clear value) | +| Events collapse to anchor per kind, no per-key attribution | Keys are secrets — not appropriate as analytics tags; LD provides better tagging mechanisms | Per-key attribution (would multiply event machinery N×) | +| Diagnostic events keep verbatim-proxy behavior | Preserves operational debug value (which SDK reported); minimal code change | Collapse diagnostic to anchor (loses debug signal); metadata-header (long-term direction, out of Phase 1 scope) | +| `ReconcileCredentials` API replaces `UpdateCredential` everywhere | Atomic semantics; single API surface; no external consumers to preserve | Keep both methods (two ways to do the same thing); stateful batching (non-idiomatic Go) | +| Hard-fail on malformed payload (Q6 working assumption) | Loud, safe, atomic | Soft-fall-back to `sdkKeys[0]` (silent, order-dependent); refuse to serve until next valid update (disruptive) | +| Order of operations: add → re-anchor → remove (Q9 working assumption) | Accepted set is a superset during transition; downstream survives | Remove first (downstream-availability window); concurrent (race-prone); atomic batch (atomicity breaks at goroutine boundary) | +| Manual config stays single-key in Phase 1 | Same trusted-source reasoning as above | Verify-on-startup, opt-in unsafe flag (rejected for the same reasons in §1) | + +--- + +## 14. Open questions (pending offline confirmation) + +These have working assumptions; Aaron is confirming with the team before lock-in. None block design or initial development. + +- **Q5**: RAC propagation SLA for `sdkKey.value` changes. *Working assumption*: real-time via SSE push. +- **Q6**: Behavior on malformed RAC payload. *Working assumption*: hard-fail, preserve previous state. +- **Q7**: Legacy `sdkKey.expiring{}` vs per-key `expiry` disagreement policy. *Working assumption*: trust the array. +- **Q8**: Does relay track per-credential downstream connections for targeted disconnect? *Working assumption*: yes (in `envStreams`); verify in code as T1.c pre-work. +- **Q11**: Customer downgrade story (rolling relay back from Phase 1). *Working assumption*: surface in release notes; no relay-side mitigation needed. + +See [`phase1-questions.md`](../../docs/agents/phase1-questions.md) in the design worktree for full context per question. (That file is not on this feature branch — it lives in the design worktree.) + +--- + +## 15. Glossary + +- **Anchor**: the SDK key the singular `sdkKey.value` field designates. Used for the upstream connection and as the event-dispatcher's stored `authKey`. +- **Accepted set**: all SDK keys + mobile keys + env IDs an environment will accept for downstream-SDK authentication. Includes the anchor. +- **Identifier** (wire `key`): the non-secret human-readable name of a credential. Used in API paths and status display. +- **Credential / value** (wire `value`): the actual secret string (e.g. `sdk-xxxx-...`). What relay's existing types `SDKKey`/`MobileKey`/`SDKCredential` refer to. +- **RAC** (Relay Auto Config): the push channel by which LaunchDarkly delivers environment configuration to enterprise relays. SSE over HTTPS. +- **Offline archive**: a file generated by LaunchDarkly tooling carrying the same `EnvironmentRep` shape. Reloaded periodically. +- **Re-anchor**: swap the upstream SDK client when `sdkKey.value` changes. Single trigger for the swap mechanism. +- **Trusted source**: a LaunchDarkly-generated configuration source (RAC or offline archive). Guaranteed to carry only the environment's real keys. diff --git a/docs/concurrent-keys/phase1-plan.md b/docs/concurrent-keys/phase1-plan.md new file mode 100644 index 00000000..0bad9563 --- /dev/null +++ b/docs/concurrent-keys/phase1-plan.md @@ -0,0 +1,416 @@ +# Phase 1 — Concurrent SDK Keys in Relay Proxy: Plan + +**Epic**: [SDK-2453](https://launchdarkly.atlassian.net/browse/SDK-2453) +**Companion**: [`phase1-design.md`](./phase1-design.md) (architecture, decisions, model) + +This document covers *how* we ship Phase 1: branching, sequencing, tasks, dependencies, estimates, test strategy, and rollout. + +--- + +## 1. Branching strategy + +**Long-lived feature branch off v8: `feat/concurrent-keys`.** Team convention for feature branches across LD repos — recognizable to other team members. + +- All sub-task PRs target `feat/concurrent-keys`, not v8 directly. +- Sub-PR branches follow Aaron's convention: `aaronz//`, where `` is the **specific sub-task ticket** (e.g. `SDK-2521` for T1.0), **not the epic SDK-2453**. Example: `aaronz/SDK-2521/T1.0-rotate-with-grace-mobile-fix`. +- **Regularly merge `v8` into `feat/concurrent-keys`** (weekly) to surface conflicts incrementally rather than at the end. +- **Final merge** `feat/concurrent-keys` → v8 happens when the feature is fully ready and verified, as a single feat commit. + +### Worktree + +```bash +git worktree add ../ld-relay-wt-feat-concurrent-keys -b feat/concurrent-keys v8 +``` + +This document is committed in that worktree at `docs/concurrent-keys/phase1-plan.md`. + +--- + +## 2. Wave breakdown + +Three waves. Wave 3 is the final merge-forward to v9, possibly weeks or months after Wave 2 completes. + +| Wave | Theme | When | +|---|---|---| +| **Wave 1** | Foundations: PoC, data structures, wire types, test infrastructure | Immediately, multiple sub-tasks in parallel | +| **Wave 2** | Core implementation: API surface change, re-anchor mechanism, peripheral re-wiring, end-to-end integration | After PoC findings + Wave 1 data structures land | +| **Wave 3** | Merge-forward to v9 | Possibly weeks/months after Wave 2, depending on v8 production deploy timing | + +The "single-key behavior unchanged at every PR boundary" invariant is the load-bearing testable property. Every sub-PR must preserve it. + +--- + +## 3. Task list + +Each task has: ticket name, files touched, dependencies, estimates. Acceptance criteria live in the JIRA ticket; rationale lives in [`phase1-design.md`](./phase1-design.md). + +### Wave 1 + +| Task | Files | Depends on | Human | AI agent | +|---|---|---|---|---| +| **T0** — Re-anchoring PoC | `internal/relayenv/` (new test files) | — | 3-5 days | 1-2 days (with iteration) | +| **T1.0** — Remove `RotateWithGrace` mobile-key panic | `internal/credential/rotator.go` | — | 0.5 day | 30 min - 1 hr | +| **T1.a** — Add Rotator accepted-set data structures | `internal/credential/rotator.go`, `credential.go` | T1.0 | 1-2 days | 1-2 hrs | +| **T3.a** — Extend `EnvironmentRep` + verify | `internal/envfactory/*`, possibly `archive_reader.go` | — | 1-2 days | 1-2 hrs | +| **T5.a** — Integration test harness | New test infrastructure dir | — | 2-3 days | 2-3 hrs | +| **T5.b** — Events payload regression test + baseline | `internal/events/*_test.go` (new) | — | 1-2 days | 1-2 hrs | + +**Wave 1 total**: 8.5-14.5 human days; 6-10 AI agent hours. + +### Wave 2 + +| Task | Files | Depends on | Human | AI agent | +|---|---|---|---|---| +| **T1.b** — `ReconcileCredentials` API + migrate call sites + remove `UpdateCredential` | `internal/relayenv/env_context*.go`, both action handlers, tests | T0, T1.a | 2-3 days | 2-4 hrs | +| **T1.c** — Generalize cleanup ticker for per-key expiry + mobile-key disconnects | `internal/credential/rotator.go`, `env_context_impl.go` | T1.b, Q8 verified | 2-3 days | 2-3 hrs | +| **T2.a** — `addCredential` anchor-only client | `internal/relayenv/env_context_impl.go` | T0, T1.b | 1-2 days | 1-2 hrs | +| **T2.b** — `GetClient()` returns anchor | `internal/relayenv/env_context_impl.go` | T2.a | 0.5 day | 30 min - 1 hr | +| **T2.c** — Re-anchor mechanism per PoC findings | `internal/relayenv/env_context_impl.go` | T0, T2.a | 3-5 days | 3-5 hrs | +| **T2.d** — Big-segment sync + `httpconfig` from anchor | `internal/relayenv/env_context_impl.go`, big-segment code | T0 | 2-3 days | 2-3 hrs | +| **T2.e** — Handler fan-out optimization | `internal/relayenv/env_context_impl.go`, stream provider interface | T2.c | 2-3 days | 2-3 hrs | +| **T3.b** — Shared reconcile helper | New helper in `internal/envfactory/` | T1.b | 2-3 days | 2-3 hrs | +| **T3.c** — Wire RAC + offline handlers | `relay/autoconfig_actions.go`, `relay/filedata_actions.go` | T3.a, T3.b, T1.b | 2-3 days | 2-3 hrs | +| **T4** — Status endpoint arrays | `internal/api/status_reps.go`, `relay/endpoints_status.go` | T1.b, T3.c | 1-2 days | 1-2 hrs | + +**Wave 2 total**: 18-30 human days; 18-32 AI agent hours. + +### Wave 3 + +| Task | Files | Depends on | Human | AI agent | +|---|---|---|---|---| +| **T5.e** — Merge-forward to v9 | `internal/relayenv/*`, streaming path | All Wave 2 done | 3-7 days | 1-2 days (with iteration) | + +**Total project**: ~6-11 weeks of full-time human work (excluding the calendar gap before Wave 3, which may extend the project's wall-clock duration substantially). + +--- + +## 4. Dependency graph + +``` + ┌─→ T1.b ─┬─→ T1.c + │ ├─→ T2.a ─→ T2.b + │ │ └──→ T2.c ─→ T2.e + │ ├─→ T3.b ─→ T3.c + │ └─→ T4 +T0 (PoC) ──────────────────────────────────────── ┤ + ├─→ T2.a (PoC needed for swap mechanism) + ├─→ T2.c + └─→ T2.d + +T1.0 ─→ T1.a ─────────────────────────────────────┘ + +T3.a ─────────────────────────────────────────────┐ + ├─→ T3.c +T3.b ─────────────────────────────────────────────┘ + +T5.a (test harness) — supports all other tasks' tests +T5.b (events regression) — runs continuously after landing + +[all Wave 2 done] ─→ T5.e (merge-forward to v9) +``` + +**Critical path** (longest dependency chain): +T1.0 → T1.a → T1.b → T2.a → T2.c → T2.e → T5.e + +This chain alone is roughly: 0.5 + 1.5 + 2.5 + 1.5 + 4 + 2.5 + 5 = ~17 human days at the midpoint of the estimates. Other Wave 2 tasks parallelize off this critical path. + +--- + +## 5. Implementation notes per task + +For each task, JIRA tickets carry the full acceptance criteria. Below are *notes that don't fit cleanly into a ticket* — design rationale, code references, things to watch for. + +### T0 — Re-anchoring PoC + +The PoC validates the swap mechanism that T2.c will implement. It is *the* prerequisite — without it, T2 is speculation. The PoC's deliverable is durable test code that survives into T2. + +Seven hypotheses to validate (each becomes a test): +1. Two clients sharing a `storeAdapter` don't corrupt store invariants. +2. Downstream SSE connections tolerate the swap. +3. Big-segment sync keeps working after re-anchor (or identify what re-wiring is needed). +4. `httpconfig` stays functional after re-anchor. +5. Settle order of operations (start-new → swap pointer → close-old vs. alternatives). +6. Behavior during the swap window. +7. Failure modes: new client init fails. + +### T1.0 — Remove `RotateWithGrace` mobile-key panic + +Today: `rotator.go:168-169` panics with `"programmer error: mobile keys do not support deprecation"`. The panic is a guard against an unsupported API state (the data model has no slot for an expiring mobile key). Removing the panic is small; the slot is added by T1.a. + +### T1.a — Rotator data structures + +Internal fields only. No API change. Existing public methods (`SDKKey()`, `GetCredentials()`, etc.) continue to return what they return today by reading from the new internal state where the single primary maps to a one-element set. + +Reviewer-friendly comment to add at the top of the new fields: `// Consumed by T1.b (ReconcileCredentials API). See docs/concurrent-keys/phase1-design.md §6.2.` + +### T1.b — `ReconcileCredentials` API + +The new method replaces `UpdateCredential` *everywhere* — both call sites migrate in this same PR, and `UpdateCredential` + supporting types are removed. There are no external consumers to preserve. + +Today's API surface (to be removed): + +```go +// internal/relayenv/env_context.go:80-85 +UpdateCredential(update *CredentialUpdate) + +// internal/relayenv/env_context.go:27-36 +type CredentialUpdate struct { + primary credential.SDKCredential + deprecated config.SDKKey + expiry time.Time + now time.Time +} +``` + +The new API (bikeshed the exact signature at PR time; this is illustrative): + +```go +ReconcileCredentials(newSet AcceptedSet, anchor credential.SDKCredential) error +``` + +`AcceptedSet` carries the full new state (server keys + mobile keys with optional per-key expiry). The implementation owns the order of operations (`add → re-anchor → remove`) internally; callers don't sequence. + +### T1.c — Cleanup ticker + +Generalize `cleanupExpiredCredentials` (called from `StepTime`) to walk the entire accepted set per kind and drop entries whose `expiry` has passed. The downstream-disconnect logic must handle mobile-key disconnects, not just SDK-key ones. **Q8 pre-work**: verify in code that `envStreams` (or an adjacent component) maintains per-credential downstream connection lists. If not, the per-key targeted disconnect needs additional infrastructure — scope expansion. + +### T2.a — `addCredential` anchor-only client + +The switch case at `env_context_impl.go:448-463` currently calls `startSDKClient` for any `config.SDKKey`. Phase 1 narrows this to "only the anchor calls `startSDKClient`." Non-anchor server keys get handlers + `envStreams` + lookup mapping but no upstream client. Mobile keys and env IDs already behave this way. + +### T2.b — `GetClient()` returns anchor + +`GetClient()` at `env_context_impl.go:580-594` already returns `c.clients[c.keyRotator.SDKKey()]`. With anchor-only client construction (T2.a), this becomes "return the only client." Verify behavior in tests; small change. + +### T2.c — Re-anchor mechanism + +The big one. Implements whatever order-of-ops and component-rewiring the PoC settled on. Per the §7 design analysis, the swap consists of: + +1. Start new upstream client on the new anchor's SDK key. +2. Wait for it to initialize (the data store is shared; the new client feeds the same store). +3. Atomically swap the rotator's anchor pointer (so `GetClient()` returns the new client). +4. Call `ReplaceCredential` on event dispatcher and metrics publisher. +5. Re-wire big-segment sync (mechanism TBD per PoC). +6. Verify `httpconfig` continues to work (likely no-op per PoC). +7. Close the old upstream client. + +PoC failure modes inform the recovery logic. + +### T2.d — Big-segment sync + `httpconfig` from anchor + +These are the two construction-time wirings. Either: +- Refactor big-segment sync to be re-wireable (add a method to point it at a new SDK key), or +- Recreate the big-segment sync component on each re-anchor (heavier but simpler). + +`httpconfig` is mostly TLS / proxy / event-base-uri config — key-independent — but verify in PoC. + +### T2.e — Handler fan-out optimization + +Refactor the handler-building loop at `env_context_impl.go:268-277`. Today: per `(credential, filter, stream provider)`. After: per `(filter, stream provider)`, with the handler resolving the credential from the request at serving time. + +At Block-scale (50 credentials × 10 filters × 4 stream providers), this is the difference between 2,000 handlers per env and 40 per env. See §6 below for the math. + +### T3.a — `EnvironmentRep` extension + +Add the new array fields and the canonical wire-types comment (see [`phase1-design.md`](./phase1-design.md) §5 for the comment text). Verify `DisallowUnknownFields` is not used in the env-parse path (additive-payload guarantee depends on this). Check whether `archive_reader.go` does its own parsing or consumes `EnvironmentRep` directly (T3.a's scope expands if it parses on its own). + +### T3.b — Shared reconcile helper + +A new helper (in `internal/envfactory/` or similar) that both `autoconfig_actions.go` and `filedata_actions.go` call. Responsibilities: +- Diff the old accepted set against the new one (set-keyed by `value`). +- Detect re-anchor (`sdkKey.value` changed). +- Compute the ordered operation list: `add → re-anchor → remove`. +- Hard-fail if the payload is malformed (anchor `value` not in `sdkKeys[]`). +- Treat the legacy `sdkKey.expiring{}` field as write-only — read only the array. + +### T3.c — Wire both action handlers + +Replace `UpdateCredential` calls with the new `ReconcileCredentials` API, via the shared helper. RAC handler and offline handler updates land in one PR (separate commits per Aaron's preference). + +Test matrix (covered in T3.c's acceptance criteria): +- Add a new key +- Set `expiry` on a non-anchor key +- Set `expiry` on the anchor (re-anchor triggered) +- Remove a key (omit from next patch) +- Rename a key (same `value`, different `key` identifier — no-op for creds) +- De-expiry (remove `expiry` on existing entry — cancel scheduled drop) +- Mixed patch (add + re-anchor + remove) +- Partial-failure reconcile (preserves previous state) + +### T4 — Status endpoint arrays + +Add `sdkKeys` / `mobileKeys` array fields to the env status response. Each entry: non-secret `Key` identifier + obscured `Value` (via `sdks.ObscureKey`) + optional `Expiry`. Keep scalar `sdkKey` / `mobileKey` — they now represent the **anchor** specifically. Keep `expiringSdkKey` for default-rotation back-compat. + +Stable ordering of array entries: anchor first, then identifier-alphabetical. Predictable for tooling consumers. + +Arrays are *present but empty* (not omitted) for single-key envs. + +### T5.a — Integration test harness + +Build a reusable harness: +- **RAC mock**: emits captured payloads, supports `put`/`patch`/`delete` event sequences. +- **Downstream SDK simulator**: simulates an SDK connecting with a credential and consuming a stream. +- **Archive fixture loader**: loads offline-mode archives from disk for the filedata path. + +The harness lands as Wave 1 infrastructure; scenarios accumulate as acceptance tests in the sub-tasks that introduce each feature. + +### T5.b — Events payload regression test + +Capture upstream payloads from v8 under realistic SDK traffic. Assert post-Phase-1 payloads are structurally identical *except* for the credential field. Catches accidental schema drift throughout the project. + +### T5.e — Merge-forward to v9 + +Not a `git merge`. Real integration work resolving FDv2 ↔ Phase 1 interactions in `env_context_impl.go` and the streaming path. v9 has FDv2 in it, which touches the same files Phase 1 changes most heavily. Validate against v9's existing test suite plus a subset of Phase 1 tests adapted for v9. + +Timing: possibly weeks or months after Wave 2 completes, depending on when v8 ships to production. + +--- + +## 6. Handler fan-out optimization (T2.e) — the math + +The optimization observes: today, relay builds one HTTP handler per `(credential, filter, stream provider)` triple. All handlers in the same `(filter, provider)` slot are byte-identical except for the credential baked in at construction. If we look up the credential from the request at serving time, we share one handler per `(filter, provider)`. + +Today: `handlers per env = C × F × P` where C = credentials, F = filters+1, P = stream providers. +After: `handlers per env = F × P`. + +| Customer profile | C | F | P | Unoptimized | Optimized | +|---|---|---|---|---|---| +| Single-key today (baseline) | 3 | 1 | 4 | 12 | 4 | +| Mid-market Phase 1 multi-key | 8 | 1 | 4 | 32 | 4 | +| **Block-scale (multi-key + multi-filter)** | **50** | **10** | **4** | **2,000 per env** | **40 per env** | + +At ~500 bytes per handler closure, Block-scale unoptimized is ~5 MB; optimized is ~100 KB. Memory itself isn't catastrophic, but secondary costs (setup time on every reconcile, GC pressure, per-request lookup overhead) add up. Block is the named customer driver for this project; shipping unoptimized risks regressing memory characteristics for the customer the project is meant to help. + +We're not gating T2.e on an empirical memory benchmark — the napkin math is sufficient justification. + +--- + +## 7. Test strategy + +### Per-PR + +Every sub-PR runs the full existing test suite via existing CI. The "single-key behavior unchanged" invariant is the testable property. + +**Code-review norm** (replaces the dropped T5.d CI job): feature-branch PRs must run the full test suite. Any test that needs to be removed or modified during Phase 1 must be explicitly justified in the PR description. + +### Distributed tests + +Each sub-task's acceptance criteria include unit and scoped-integration tests for that sub-task. Examples: + +- T1.0: panic-removal unit test. +- T1.a: data-structure tests. +- T1.b: `ReconcileCredentials` unit + integration tests. +- T1.c: cleanup-ticker tests, including per-key expiry and mobile-key disconnect. +- T2.c: re-anchor integration tests (evolved from PoC). +- T3.a: parse format tests + old-relay back-compat test + `DisallowUnknownFields` verification. +- T3.c: the reconcile scenario matrix (add, set-expiry, remove, rename, de-expiry, mixed, partial failure). +- T4: status endpoint scenario tests. + +### Cross-cutting tests + +These live in T5 and run continuously: + +- **T5.a (test harness)**: enables the per-sub-task tests above. +- **T5.b (events payload regression)**: catches schema drift in event payloads. + +### Release-readiness checklist + +Before merging `feat/concurrent-keys` to v8 (and again before deploying to production), run through: + +1. All Wave 2 sub-tasks merged and tests passing. +2. End-to-end customer-journey integration tests pass (assembled from T5.a + per-task acceptance tests). +3. Events payload regression test (T5.b) passes against the full feature branch. +4. Single-key behavior verified identical to v8's baseline via full test suite. +5. Status endpoint manually inspected for both single-key and multi-key envs. +6. Defensive payload tests: malformed RAC payload → relay logs + preserves previous state. + +This is a *checklist*, not a discrete task. Touched at release readiness, not as a separate sub-PR. + +--- + +## 8. Rollout + +### Release notes + +Three customer-facing items to surface: + +1. **"Concurrent SDK keys are available for relays using LaunchDarkly's Relay Auto Config.** Manual configuration continues to support one SDK key, one mobile key, and one environment ID per environment. Multi-key support for manual configuration will arrive in a future major release." +2. **"Events from all SDK keys in an environment appear under the anchor key in LaunchDarkly analytics."** This is consistent with today's single-key behavior but worth calling out because the multi-key model invites the expectation that attribution would split. +3. **"Status endpoint adds `sdkKeys` and `mobileKeys` array fields** showing all accepted keys with non-secret identifiers and obscured credentials. Existing `sdkKey` and `mobileKey` scalars now represent the *anchor* key specifically (the key relay uses for its upstream connection)." + +### External follow-ups + +These are tracked but not part of Phase 1's task list: + +- **Public docs update**: the customer docs at `launchdarkly.com/docs/home/account/environment/keys` currently say *"If you are using the Relay Proxy, it can only use the default SDK key."* Phase 1 invalidates this. Aaron's team doesn't own public docs; Aaron contacts the docs-owning team when Phase 1 is close to shipping. + +### Kill switch + +Phase 1 doesn't introduce a config-level kill switch for concurrent keys. Multi-key behavior is effectively opt-in at the customer level — customers who don't create additional keys in LD's UI see zero behavior change. If a critical issue surfaces, the operational mitigation is: customer rolls back to a pre-Phase-1 build, and SDKs using non-anchor keys lose connectivity until the operator updates either the relay or the LD UI. + +### Customer downgrade story (open question for the team) + +Tracked as Q11. Working assumption: surface in release notes; no relay-side mitigation needed beyond messaging. + +--- + +## 9. Deferred items + +These are intentional non-goals, with notes on what would trigger reconsideration: + +- **Memory benchmark for T2.e** — deferred. Napkin math (§6) is sufficient justification. Revive if teammates push back without empirical data. +- **Verify-on-startup for manual-config keys** — rejected. The full reasoning is in [`phase1-design.md`](./phase1-design.md) §1 / §13. +- **Per-key event attribution** — deliberately not pursued. Would require multiplying event machinery; SDK keys are secrets and not appropriate as analytics tags. Long-term path if ever needed: a non-secret metadata header on diagnostic events. Out of Phase 1 scope. +- **Multi env-ID support** — out of scope for Phase 1, mirroring the backend tech spec which also defers client-side ID migration. +- **Phase 2 mega-stream design** — out of scope for this plan. Phase 2 will get its own design doc. + +--- + +## 10. JIRA structure + +``` +SDK-2453 (Epic) — Relay Proxy Multi Keys Support +├── T0 Re-anchoring PoC [Story] +├── T1 Generalize the credential model [Story] +│ ├── T1.0 Remove RotateWithGrace mobile-key panic [Sub-task] +│ ├── T1.a Add Rotator accepted-set data structures [Sub-task] +│ ├── T1.b ReconcileCredentials API + migrate + remove [Sub-task] +│ └── T1.c Generalize cleanup ticker [Sub-task] +├── T2 Decouple upstream-client lifecycle [Story] +│ ├── T2.a addCredential anchor-only client [Sub-task] +│ ├── T2.b GetClient returns anchor's client [Sub-task] +│ ├── T2.c Re-anchor mechanism per PoC [Sub-task] +│ ├── T2.d Big-segment + httpconfig from anchor [Sub-task] +│ └── T2.e Handler fan-out optimization [Sub-task] +├── T3 Plumb N keys from trusted sources [Story] +│ ├── T3.a Extend EnvironmentRep + verify [Sub-task] +│ ├── T3.b Shared reconcile helper [Sub-task] +│ └── T3.c Wire RAC + offline handlers [Sub-task] +├── T4 Status endpoints [Task] +└── T5 Tests + merge-forward [Story] + ├── T5.a Integration test harness [Sub-task] + ├── T5.b Events payload regression test [Sub-task] + └── T5.e Merge-forward to v9 [Sub-task] +``` + +**Wave labels**: `wave-1`, `wave-2`, `wave-3` on every sub-task (and `wave-1` on T0 since it has no sub-tasks). + +**Dependencies**: modeled via JIRA `blocks` links. See §4 above for the full graph. + +--- + +## 11. Quick reference + +| Question | Answer | +|---|---| +| Feature branch? | `feat/concurrent-keys` off v8 | +| Sub-PR branches? | `aaronz//` off the feature branch (use the specific sub-task ticket ID, not the epic SDK-2453) | +| Where do canonical docs live? | This file + `phase1-design.md` in `docs/concurrent-keys/` on the feature branch | +| Where do working notes live? | `docs/agents/phase1-*.md` in the design worktree (gitignored, not on this branch) | +| How is ordering enforced within a `keys change` event? | Add → re-anchor → remove (atomic) | +| What triggers re-anchor? | `sdkKey.value` changed | +| Trusted sources for additional keys? | RAC + offline archive only. Manual config = single-key. | +| Events attribution? | Anchor per kind (collapse). No per-key attribution. | +| Test invariant at every PR boundary? | Single-key behavior identical to v8 | +| Where's the rationale for X decision? | See `phase1-design.md` §13 (Recorded decisions) | +| Open questions still pending? | `phase1-design.md` §14 — Q5, Q6, Q7, Q8, Q11 | From 65fddf17dccba45e304ad92ee90727c1f6de02c0 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 17 Jun 2026 11:24:11 -0700 Subject: [PATCH 02/66] feat(credential): remove RotateWithGrace mobile-key panic (#701) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mobile key rotation no longer panics when RotateWithGrace is called with a non-nil grace period. The outgoing key can be recorded in a new deprecatedMobileKeys map (key → expiry), mirroring the SDK-key deprecation model at a minimal level. updateMobileKey now takes grace *GracePeriod: nil or already-expired grace still revokes the previous key immediately (same as before for existing callers). A valid grace period only stores the old key’s expiry; StepTime does not expire deprecated mobile keys yet (planned in T1.c). Re-promoting a key clears it from deprecatedMobileKeys. --- internal/credential/rotator.go | 54 +++++++++++------ internal/credential/rotator_test.go | 94 +++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 18 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index da9b167f..1bc2e62b 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -12,9 +12,14 @@ import ( type Rotator struct { loggers ldlog.Loggers - // There is only one mobile key active at a given time; it does not support a deprecation period. + // There is only one mobile key active at a given time. primaryMobileKey config.MobileKey + // deprecatedMobileKeys stores mobile keys being phased out with a grace period, keyed + // by credential value with the associated expiry time. StepTime does not yet act on + // these entries — that is deferred to T1.c (generalize cleanup ticker). + deprecatedMobileKeys map[config.MobileKey]time.Time + // There is only one environment ID active at a given time, and it won't actually be rotated. The mechanism is // here to allow setting it in a deferred manner. primaryEnvironmentID config.EnvironmentID @@ -42,8 +47,9 @@ type InitialCredentials struct { // contains no credentials and can optionally be initialized via Initialize. func NewRotator(loggers ldlog.Loggers) *Rotator { r := &Rotator{ - loggers: loggers, - deprecatedSdkKeys: make(map[config.SDKKey]time.Time), + loggers: loggers, + deprecatedSdkKeys: make(map[config.SDKKey]time.Time), + deprecatedMobileKeys: make(map[config.MobileKey]time.Time), } return r } @@ -156,19 +162,16 @@ func NewGracePeriod(key config.SDKKey, expiry time.Time, now time.Time) *GracePe return &GracePeriod{key, expiry, now} } -// RotateWithGrace sets a new primary credential while deprecating a previous credential. The grace -// parameter may be nil to immediately revoke the previous credential. -// It is invalid to specify a grace period when the credential being rotate is a mobile key or -// environment ID. +// RotateWithGrace sets a new primary credential while deprecating the previous one. When grace is nil +// the outgoing credential is immediately revoked. It is invalid to specify a grace period for an +// environment ID. For mobile keys, a non-nil grace period stores the expiry for the outgoing key; +// the cleanup ticker is responsible for acting on it. func (r *Rotator) RotateWithGrace(primary SDKCredential, grace *GracePeriod) { switch primary := primary.(type) { case config.SDKKey: r.updateSDKKey(primary, grace) case config.MobileKey: - if grace != nil { - panic("programmer error: mobile keys do not support deprecation") - } - r.updateMobileKey(primary) + r.updateMobileKey(primary, grace) case config.EnvironmentID: if grace != nil { panic("programmer error: environment IDs do not support deprecation") @@ -194,21 +197,36 @@ func (r *Rotator) updateEnvironmentID(envID config.EnvironmentID) { } } -func (r *Rotator) updateMobileKey(mobileKey config.MobileKey) { - if mobileKey == r.MobileKey() { - return - } +// updateMobileKey sets a new primary mobile key. When grace is nil the outgoing key is +// immediately revoked; when non-nil its expiry is stored in deprecatedMobileKeys. +// StepTime does not yet act on deprecatedMobileKeys — that is deferred to T1.c. +func (r *Rotator) updateMobileKey(mobileKey config.MobileKey, grace *GracePeriod) { r.mu.Lock() defer r.mu.Unlock() + if mobileKey == r.primaryMobileKey { + return + } previous := r.primaryMobileKey r.primaryMobileKey = mobileKey + delete(r.deprecatedMobileKeys, mobileKey) r.additions = append(r.additions, mobileKey) - if previous.Defined() { + if !previous.Defined() { + r.loggers.Infof("New primary mobile key is %s", mobileKey.Masked()) + return + } + if grace == nil { r.expirations = append(r.expirations, previous) r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", previous.Masked(), mobileKey.Masked()) - } else { - r.loggers.Infof("New primary mobile key is %s", mobileKey.Masked()) + return + } + if grace.Expired() { + r.loggers.Infof("Deprecated mobile key %s already expired at %v; revoking immediately", previous.Masked(), grace.expiry) + r.expirations = append(r.expirations, previous) + return } + r.deprecatedMobileKeys[previous] = grace.expiry + r.loggers.Infof("Mobile key %s was marked for deprecation with an expiry at %v, new primary mobile key is %s", + previous.Masked(), grace.expiry, mobileKey.Masked()) } func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 1cb23336..d06379de 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -231,3 +231,97 @@ func TestSDKKeyExpiredInThePastIsNotAdded(t *testing.T) { assert.ElementsMatch(t, []SDKCredential{primaryKey}, additions) assert.Empty(t, expirations) } + +func TestRotateWithGraceMobileKey(t *testing.T) { + t.Run("does not panic with non-nil grace period", func(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + mob1 := config.MobileKey("mob1") + mob2 := config.MobileKey("mob2") + + start := time.Unix(10000, 0) + expiry := start.Add(1 * time.Hour) + + rotator.Initialize([]SDKCredential{mob1}) + + // GracePeriod.key is SDK-key typed; pass a zero value since mobile-key rotation + // does not use that identifier field. + assert.NotPanics(t, func() { + rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), expiry, start)) + }) + + assert.Equal(t, mob2, rotator.MobileKey()) + + // mob2 is a new addition; mob1 is in the deprecated set (not yet expired), + // so it should not appear as an expiration here. Cleanup is deferred to T1.c. + additions, expirations := rotator.StepTime(start) + assert.ElementsMatch(t, []SDKCredential{mob2}, additions) + assert.Empty(t, expirations) + }) + + t.Run("immediately revokes outgoing key when grace period is already expired", func(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + mob1 := config.MobileKey("mob1") + mob2 := config.MobileKey("mob2") + + expiry := time.Unix(10000, 0) + now := expiry.Add(1 * time.Hour) // now is after expiry + + rotator.Initialize([]SDKCredential{mob1}) + + rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), expiry, now)) + + assert.Equal(t, mob2, rotator.MobileKey()) + + additions, expirations := rotator.StepTime(now) + assert.ElementsMatch(t, []SDKCredential{mob2}, additions) + assert.ElementsMatch(t, []SDKCredential{mob1}, expirations) + }) + + t.Run("immediately revokes outgoing key when grace is nil", func(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + mob1 := config.MobileKey("mob1") + mob2 := config.MobileKey("mob2") + + rotator.Initialize([]SDKCredential{mob1}) + rotator.RotateWithGrace(mob2, nil) + + assert.Equal(t, mob2, rotator.MobileKey()) + + additions, expirations := rotator.StepTime(time.Now()) + assert.ElementsMatch(t, []SDKCredential{mob2}, additions) + assert.ElementsMatch(t, []SDKCredential{mob1}, expirations) + }) + + t.Run("re-promoting a deprecated key removes it from the deprecated set", func(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + mob1 := config.MobileKey("mob1") + mob2 := config.MobileKey("mob2") + + start := time.Unix(10000, 0) + expiry := start.Add(1 * time.Hour) + + rotator.Initialize([]SDKCredential{mob1}) + + // Rotate mob1 → mob2 with grace; mob1 enters deprecatedMobileKeys. + rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), expiry, start)) + rotator.StepTime(start) + + // Rotate back mob2 → mob1; mob1 should be promoted out of the deprecated set. + rotator.RotateWithGrace(mob1, nil) + + assert.Equal(t, mob1, rotator.MobileKey()) + + // mob1 should appear only as an addition, not also as an expiration. + additions, expirations := rotator.StepTime(start) + assert.ElementsMatch(t, []SDKCredential{mob1}, additions) + assert.ElementsMatch(t, []SDKCredential{mob2}, expirations) + }) +} From ecc9635c5223b50d669b2b00699ece87def954fd Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 17 Jun 2026 13:43:44 -0700 Subject: [PATCH 03/66] test(events): add credential routing tests for EventDispatcher (#703) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds internal/events/event_payload_regression_test.go with regression tests for EventDispatcher Authorization routing when the incoming SDK request’s credential differs from the dispatcher’s stored anchor (authKey). --- .../events/event_payload_regression_test.go | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 internal/events/event_payload_regression_test.go diff --git a/internal/events/event_payload_regression_test.go b/internal/events/event_payload_regression_test.go new file mode 100644 index 00000000..2b2d0994 --- /dev/null +++ b/internal/events/event_payload_regression_test.go @@ -0,0 +1,137 @@ +// Credential routing regression tests for the EventDispatcher. +// +// Invariants: +// 1. Analytics events are always forwarded upstream under the anchor credential +// (EventDispatcher's stored authKey) — never under the credential that arrived +// on the incoming SDK request. +// 2. Diagnostic events proxy the incoming request's Authorization header verbatim; +// the anchor credential is not used. +// 3. After ReplaceCredential is called (anchor rotation), analytics switches to the +// new anchor immediately; diagnostic forwarding is unaffected. +package events + +import ( + "net/http/httptest" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/basictypes" + st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + + ldevents "github.com/launchdarkly/go-sdk-events/v3" + helpers "github.com/launchdarkly/go-test-helpers/v3" + "github.com/launchdarkly/go-test-helpers/v3/httphelpers" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func requireUpstreamRequest(t *testing.T, requestsCh <-chan httphelpers.HTTPRequestInfo) httphelpers.HTTPRequestInfo { + t.Helper() + return helpers.RequireValue(t, requestsCh, time.Second) +} + +// TestAnalyticsUpstreamUsesAnchorCredential verifies that analytics events are forwarded +// upstream under the dispatcher's stored anchor credential, even when the incoming SDK +// request carries a different Authorization header. +func TestAnalyticsUpstreamUsesAnchorCredential(t *testing.T) { + eventRelayTest(t, st.EnvMain, config.EventsConfig{}, func(p eventRelayTestParams) { + headers := headersWithEventSchema(CurrentEventsSchemaVersion) + // Incoming request carries a non-anchor key — it must not reach the upstream. + headers.Set("Authorization", "sdk-non-anchor-key-must-not-reach-upstream") + + handler := p.dispatcher.GetHandler(basictypes.ServerSDK, ldevents.AnalyticsEventDataKind) + require.NotNil(t, handler) + w := httptest.NewRecorder() + handler(w, st.BuildRequest("POST", "/", []byte(eventPayloadForVerbatimOnly), headers)) + assert.Equal(t, 202, w.Result().StatusCode) + + p.dispatcher.flush() + r := requireUpstreamRequest(t, p.requestsCh) + + assert.Equal(t, string(st.EnvMain.Config.SDKKey), r.Request.Header.Get("Authorization"), + "analytics upstream must carry the anchor credential, not the incoming request credential") + }) +} + +// TestDiagnosticUpstreamProxiesIncomingCredential verifies that diagnostic events proxy +// the incoming request's Authorization header verbatim to the upstream, not the anchor. +func TestDiagnosticUpstreamProxiesIncomingCredential(t *testing.T) { + const sdkAuth = "sdk-original-diagnostic-client-auth" + + eventRelayTest(t, st.EnvMain, config.EventsConfig{}, func(p eventRelayTestParams) { + headers := headersWithEventSchema(0) + headers.Set("Authorization", sdkAuth) + + handler := p.dispatcher.GetHandler(basictypes.ServerSDK, ldevents.DiagnosticEventDataKind) + require.NotNil(t, handler) + w := httptest.NewRecorder() + handler(w, st.BuildRequest("POST", "/", []byte(eventPayloadForVerbatimOnly), headers)) + assert.Equal(t, 202, w.Result().StatusCode) + + r := requireUpstreamRequest(t, p.requestsCh) + + assert.Equal(t, sdkAuth, r.Request.Header.Get("Authorization"), + "diagnostic upstream must proxy the incoming Authorization header verbatim") + assert.NotEqual(t, string(st.EnvMain.Config.SDKKey), r.Request.Header.Get("Authorization"), + "diagnostic upstream must not use the anchor credential") + }) +} + +// TestCredentialRoutingAfterReplaceCredential verifies that after ReplaceCredential is +// called (anchor rotation), analytics events use the new anchor while diagnostic events +// continue to proxy the original incoming authorization. +func TestCredentialRoutingAfterReplaceCredential(t *testing.T) { + newAnchorKey := config.SDKKey(string(st.EnvMain.Config.SDKKey) + "-rotated") + const sdkDiagAuth = "sdk-original-client-that-sent-diagnostic" + + eventRelayTest(t, st.EnvMain, config.EventsConfig{}, func(p eventRelayTestParams) { + analyticsHandler := p.dispatcher.GetHandler(basictypes.ServerSDK, ldevents.AnalyticsEventDataKind) + diagnosticHandler := p.dispatcher.GetHandler(basictypes.ServerSDK, ldevents.DiagnosticEventDataKind) + require.NotNil(t, analyticsHandler) + require.NotNil(t, diagnosticHandler) + + sendAnalytics := func() httphelpers.HTTPRequestInfo { + analyticsHandler( + httptest.NewRecorder(), + st.BuildRequest("POST", "/", []byte(eventPayloadForVerbatimOnly), headersWithEventSchema(CurrentEventsSchemaVersion)), + ) + p.dispatcher.flush() + return requireUpstreamRequest(t, p.requestsCh) + } + + sendDiagnostic := func() httphelpers.HTTPRequestInfo { + headers := headersWithEventSchema(0) + headers.Set("Authorization", sdkDiagAuth) + diagnosticHandler( + httptest.NewRecorder(), + st.BuildRequest("POST", "/", []byte(eventPayloadForVerbatimOnly), headers), + ) + // Diagnostic events are forwarded immediately; no flush needed. + return requireUpstreamRequest(t, p.requestsCh) + } + + // Before rotation: analytics uses the original anchor. + r := sendAnalytics() + assert.Equal(t, string(st.EnvMain.Config.SDKKey), r.Request.Header.Get("Authorization"), + "before rotation: analytics must use original anchor") + + // Rotate the anchor. + p.dispatcher.ReplaceCredential(newAnchorKey) + + // After rotation: analytics must switch to the new anchor. + r = sendAnalytics() + assert.Equal(t, string(newAnchorKey), r.Request.Header.Get("Authorization"), + "after rotation: analytics must use new anchor credential") + assert.NotEqual(t, string(st.EnvMain.Config.SDKKey), r.Request.Header.Get("Authorization"), + "after rotation: analytics must not still use old anchor credential") + + // After rotation: diagnostic must still proxy the original incoming auth. + r = sendDiagnostic() + assert.Equal(t, sdkDiagAuth, r.Request.Header.Get("Authorization"), + "after rotation: diagnostic must still proxy the original incoming authorization") + assert.NotEqual(t, string(newAnchorKey), r.Request.Header.Get("Authorization"), + "after rotation: diagnostic must not use the new anchor credential") + }) +} From 8eee9719919fe3c3ee2cb779229fff81ce2b0478 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 17 Jun 2026 13:47:35 -0700 Subject: [PATCH 04/66] test(relay): fix flaky TestAutoConfigRemovesCredentialForExpiredSDKKey (#707) Fixed an intermittent test failure in `TestAutoConfigRemovesCredentialForExpiredSDKKey` caused by a race between credential registration and relay connection mapping. --- relay/testutils_test.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/relay/testutils_test.go b/relay/testutils_test.go index e3037143..cd89c55e 100644 --- a/relay/testutils_test.go +++ b/relay/testutils_test.go @@ -133,10 +133,26 @@ func (h relayTestHelper) assertEndpointStatus( func (h relayTestHelper) awaitCredentialsUpdated(env relayenv.EnvContext, expected envfactory.EnvironmentParams) { expectedCredentials := credentialsAsSet(expected.EnvID, expected.MobileKey, expected.SDKKey) - isChanged := func() bool { - return reflect.DeepEqual(credentialsAsSet(env.GetCredentials()...), expectedCredentials) + // Poll until both env.GetCredentials() and relay's connection mappings reflect the new credentials. + // The two updates are not atomic: AddCredential runs before AddConnectionMapping, so there is a + // window where GetCredentials() shows the new key but getEnvironment() still returns an error. + isReady := func() bool { + if !reflect.DeepEqual(credentialsAsSet(env.GetCredentials()...), expectedCredentials) { + return false + } + for _, cred := range []sdkauth.ScopedCredential{ + sdkauth.New(expected.EnvID), + sdkauth.New(expected.MobileKey), + sdkauth.New(expected.SDKKey), + } { + found, err := h.relay.getEnvironment(cred) + if err != nil || found != env { + return false + } + } + return true } - require.Eventually(h.t, isChanged, time.Second, time.Millisecond*5) + require.Eventually(h.t, isReady, time.Second, time.Millisecond*5) h.assertEnvLookup(env, expected) } From 8800fc2da7b3ad8bcc572c726c56f056dd778dc7 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 17 Jun 2026 14:10:03 -0700 Subject: [PATCH 05/66] test(sharedtest): add Phase 1 integration test utilities (#705) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds reusable building blocks under internal/sharedtest so full-stack tests outside package relay can drive RAC and offline config the same way in-process relay tests already do. `relay/concurrent_keys_harness_ref_test.go` is a reference integration test covering offline archive → SDK stream and RAC mock → SDK stream paths. --- .../configsource/archive_fixture.go | 218 ++++++++++++++++++ .../sharedtest/configsource/package_info.go | 10 + internal/sharedtest/configsource/rac_mock.go | 95 ++++++++ internal/sharedtest/stream_request.go | 26 +++ relay/concurrent_keys_harness_ref_test.go | 132 +++++++++++ 5 files changed, 481 insertions(+) create mode 100644 internal/sharedtest/configsource/archive_fixture.go create mode 100644 internal/sharedtest/configsource/package_info.go create mode 100644 internal/sharedtest/configsource/rac_mock.go create mode 100644 relay/concurrent_keys_harness_ref_test.go diff --git a/internal/sharedtest/configsource/archive_fixture.go b/internal/sharedtest/configsource/archive_fixture.go new file mode 100644 index 00000000..24dfb477 --- /dev/null +++ b/internal/sharedtest/configsource/archive_fixture.go @@ -0,0 +1,218 @@ +package configsource + +import ( + "archive/tar" + "compress/gzip" + "crypto/md5" //nolint:gosec // MD5 is used only for change-detection, not authentication + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + + helpers "github.com/launchdarkly/go-test-helpers/v3" +) + +// ArchiveEnvSpec describes one environment to be written into an offline-mode archive. +// Flags and Segments follow the same JSON shape as the archive data files; values should be +// JSON-serializable objects (e.g. from ldbuilders). Both are optional. +type ArchiveEnvSpec struct { + // Rep is the EnvironmentRep written to the metadata file. + Rep envfactory.EnvironmentRep + // DataID is an opaque string stored alongside the metadata. Relay uses it to detect whether + // the data file changed across reloads. Any non-empty string works. + DataID string + // Flags is a map of flag key → JSON-serializable flag object. May be nil. + Flags map[string]any + // Segments is a map of segment key → JSON-serializable segment object. May be nil. + Segments map[string]any +} + +// ArchiveFixtureBuilder builds offline-mode archive files (.tar.gz) for use as Relay's +// FileDataSource. The archive format matches what internal/filedata.ArchiveManager expects: +// an {envID}.json metadata file, an {envID}-data.json flag/segment data file, and a checksum.md5 +// file. Call AddEnv one or more times, then WriteTempFile or WriteFile. +type ArchiveFixtureBuilder struct { + envs []ArchiveEnvSpec +} + +// NewArchiveFixtureBuilder creates an empty builder. +func NewArchiveFixtureBuilder() *ArchiveFixtureBuilder { + return &ArchiveFixtureBuilder{} +} + +// AddEnv adds an environment to the archive. Returns the builder for chaining. +func (b *ArchiveFixtureBuilder) AddEnv(spec ArchiveEnvSpec) *ArchiveFixtureBuilder { + b.envs = append(b.envs, spec) + return b +} + +// WriteTempFile writes the archive to a temporary .tar.gz file and returns its path. The file is +// removed automatically when the test ends. +func (b *ArchiveFixtureBuilder) WriteTempFile(t testing.TB) string { + t.Helper() + f, err := os.CreateTemp("", "ld-relay-archive-*.tar.gz") + if err != nil { + t.Fatalf("ArchiveFixtureBuilder: create temp file: %v", err) + } + path := f.Name() + _ = f.Close() + t.Cleanup(func() { _ = os.Remove(path) }) + b.WriteFile(t, path) + return path +} + +// WriteFile writes the archive to the given path as a .tar.gz file. +func (b *ArchiveFixtureBuilder) WriteFile(t testing.TB, path string) { + t.Helper() + + // Each environment maps to one pair of files ({envID}.json, {envID}-data.json). A duplicate + // EnvID would overwrite those files but be hashed twice in checksum.md5, producing an archive + // that fails filedata's checksum verification. Reject it rather than write a broken archive. + seen := make(map[config.EnvironmentID]bool, len(b.envs)) + for _, spec := range b.envs { + if seen[spec.Rep.EnvID] { + t.Fatalf("ArchiveFixtureBuilder: duplicate environment ID %q; each env must be added once", spec.Rep.EnvID) + } + seen[spec.Rep.EnvID] = true + } + + // Stage files in a temp directory, compute checksum, then tar.gz the result. + helpers.WithTempDir(func(dir string) { + for _, spec := range b.envs { + b.writeEnvFiles(t, dir, spec) + } + envIDs := make([]config.EnvironmentID, 0, len(b.envs)) + for _, spec := range b.envs { + envIDs = append(envIDs, spec.Rep.EnvID) + } + writeArchiveChecksum(t, dir, envIDs) + writeArchiveTarGz(t, path, dir) + }) +} + +// archiveEnvRep mirrors the unexported filedata.archiveEnvironmentRep JSON structure. +type archiveEnvRep struct { + Env envfactory.EnvironmentRep `json:"env"` + DataID string `json:"dataId"` +} + +func (b *ArchiveFixtureBuilder) writeEnvFiles(t testing.TB, dir string, spec ArchiveEnvSpec) { + t.Helper() + + // {envID}.json + metaBytes, err := json.Marshal(archiveEnvRep{Env: spec.Rep, DataID: spec.DataID}) + if err != nil { + t.Fatalf("ArchiveFixtureBuilder: marshal env metadata: %v", err) + } + writeArchiveFile(t, archiveMetadataPath(dir, spec.Rep.EnvID), metaBytes) + + // {envID}-data.json + sdkData := make(map[string]any, 2) + if len(spec.Flags) > 0 { + sdkData["flags"] = spec.Flags + } + if len(spec.Segments) > 0 { + sdkData["segments"] = spec.Segments + } + dataBytes, err := json.Marshal(sdkData) + if err != nil { + t.Fatalf("ArchiveFixtureBuilder: marshal sdk data: %v", err) + } + writeArchiveFile(t, archiveDataPath(dir, spec.Rep.EnvID), dataBytes) +} + +func writeArchiveChecksum(t testing.TB, dir string, envIDs []config.EnvironmentID) { + t.Helper() + paths := make([]string, 0, len(envIDs)*2) + for _, id := range envIDs { + paths = append(paths, archiveMetadataPath(dir, id), archiveDataPath(dir, id)) + } + sort.Strings(paths) + + h := md5.New() //nolint:gosec + for _, p := range paths { + f, err := os.Open(filepath.Clean(p)) + if err != nil { + t.Fatalf("ArchiveFixtureBuilder: open for checksum %s: %v", p, err) + } + if _, err = io.Copy(h, f); err != nil { + _ = f.Close() + t.Fatalf("ArchiveFixtureBuilder: hash %s: %v", p, err) + } + _ = f.Close() + } + writeArchiveFile(t, filepath.Join(dir, "checksum.md5"), h.Sum(nil)) +} + +func writeArchiveTarGz(t testing.TB, destPath, srcDir string) { + t.Helper() + _ = os.Remove(destPath) + f, err := os.OpenFile(filepath.Clean(destPath), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + t.Fatalf("ArchiveFixtureBuilder: create archive %s: %v", destPath, err) + } + gz := gzip.NewWriter(f) + tw := tar.NewWriter(gz) + + entries, err := os.ReadDir(srcDir) + if err != nil { + t.Fatalf("ArchiveFixtureBuilder: read staging dir: %v", err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + srcPath := filepath.Join(srcDir, entry.Name()) + fi, err := os.Stat(srcPath) + if err != nil { + t.Fatalf("ArchiveFixtureBuilder: stat %s: %v", srcPath, err) + } + hdr, err := tar.FileInfoHeader(fi, "") + if err != nil { + t.Fatalf("ArchiveFixtureBuilder: tar header for %s: %v", entry.Name(), err) + } + hdr.Name = entry.Name() // strip any directory prefix + if err = tw.WriteHeader(hdr); err != nil { + t.Fatalf("ArchiveFixtureBuilder: write tar header: %v", err) + } + src, err := os.Open(filepath.Clean(srcPath)) + if err != nil { + t.Fatalf("ArchiveFixtureBuilder: open %s: %v", srcPath, err) + } + if _, err = io.Copy(tw, src); err != nil { + _ = src.Close() + t.Fatalf("ArchiveFixtureBuilder: copy %s into tar: %v", entry.Name(), err) + } + _ = src.Close() + } + if err := tw.Close(); err != nil { + t.Fatalf("ArchiveFixtureBuilder: close tar: %v", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("ArchiveFixtureBuilder: close gzip: %v", err) + } + if err := f.Close(); err != nil { + t.Fatalf("ArchiveFixtureBuilder: close file: %v", err) + } +} + +func archiveMetadataPath(dir string, id config.EnvironmentID) string { + return filepath.Join(dir, fmt.Sprintf("%s.json", string(id))) +} + +func archiveDataPath(dir string, id config.EnvironmentID) string { + return filepath.Join(dir, fmt.Sprintf("%s-data.json", string(id))) +} + +func writeArchiveFile(t testing.TB, path string, data []byte) { + t.Helper() + if err := os.WriteFile(filepath.Clean(path), data, 0600); err != nil { + t.Fatalf("ArchiveFixtureBuilder: write %s: %v", path, err) + } +} diff --git a/internal/sharedtest/configsource/package_info.go b/internal/sharedtest/configsource/package_info.go new file mode 100644 index 00000000..ba1ac328 --- /dev/null +++ b/internal/sharedtest/configsource/package_info.go @@ -0,0 +1,10 @@ +// Package configsource contains test helpers that mock or build the sources Relay loads its +// environment configuration from: the Relay Auto Config (RAC) SSE stream and offline-mode archives. +// +// These live in sharedtest/configsource rather than sharedtest itself because they reference the +// envfactory package, which transitively imports relayenv and streams. Putting them in a subpackage +// keeps the top-level sharedtest package importable by relayenv and streams without a circular +// reference (see sharedtest/package_info.go). +// +// Non-test code should never import this package. +package configsource diff --git a/internal/sharedtest/configsource/rac_mock.go b/internal/sharedtest/configsource/rac_mock.go new file mode 100644 index 00000000..976baf74 --- /dev/null +++ b/internal/sharedtest/configsource/rac_mock.go @@ -0,0 +1,95 @@ +package configsource + +import ( + "encoding/json" + "net/http/httptest" + "testing" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/autoconfig" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + + "github.com/launchdarkly/go-test-helpers/v3/httphelpers" +) + +// RACMock is a test HTTP server that emits SSE events in the Relay Auto Config protocol format. +// Tests use it as Relay's config-stream endpoint by pointing Main.StreamURI at its URL. +// +// It is a thin, reusable wrapper over httphelpers.SSEHandler + httptest.Server with automatic +// cleanup, intended for tests outside package relay that cannot reach that package's unexported +// autoConfTest helper. Use the MakeAutoConfig*Event builders to construct events. +type RACMock struct { + // URL is the server's base URL. Point Relay's Main.StreamURI here. + URL string + server *httptest.Server + stream httphelpers.SSEStreamControl +} + +// NewRACMock creates a new RACMock SSE server. If initialEvent is non-nil it is replayed to every +// new client that connects (use this for the initial put event). Cleanup is registered with +// t.Cleanup; call Close() explicitly only if you need early teardown. +func NewRACMock(t testing.TB, initialEvent *httphelpers.SSEEvent) *RACMock { + handler, stream := httphelpers.SSEHandler(initialEvent) + server := httptest.NewServer(handler) + m := &RACMock{ + URL: server.URL, + server: server, + stream: stream, + } + t.Cleanup(m.Close) + return m +} + +// Enqueue queues an event to be delivered to the next client that connects. Use this before Relay +// has connected to ensure the event is not dropped. +func (m *RACMock) Enqueue(event httphelpers.SSEEvent) { + m.stream.Enqueue(event) +} + +// Send emits an event to all currently connected clients. Use this after Relay has connected and +// you want to trigger a live config update. +func (m *RACMock) Send(event httphelpers.SSEEvent) { + m.stream.Send(event) +} + +// Close shuts down the mock server and terminates any open SSE connections. +func (m *RACMock) Close() { + _ = m.stream.Close() + m.server.Close() +} + +// MakeAutoConfigPutEvent creates an SSE event representing a full RAC put, containing all of the +// given environments. +func MakeAutoConfigPutEvent(envs ...envfactory.EnvironmentRep) httphelpers.SSEEvent { + data := autoconfig.PutMessageData{ + Path: "/", + Data: autoconfig.PutContent{ + Environments: make(map[config.EnvironmentID]envfactory.EnvironmentRep, len(envs)), + }, + } + for _, e := range envs { + data.Data.Environments[e.EnvID] = e + } + jsonBytes, _ := json.Marshal(data) + return httphelpers.SSEEvent{Event: autoconfig.PutEvent, Data: string(jsonBytes)} +} + +// MakeAutoConfigPatchEvent creates an SSE event representing a RAC patch for a single environment. +func MakeAutoConfigPatchEvent(env envfactory.EnvironmentRep) httphelpers.SSEEvent { + repBytes, _ := json.Marshal(env) + msgBytes, _ := json.Marshal(autoconfig.PatchMessageData{ + Path: "/environments/" + string(env.EnvID), + Data: repBytes, + }) + return httphelpers.SSEEvent{Event: autoconfig.PatchEvent, Data: string(msgBytes)} +} + +// MakeAutoConfigDeleteEvent creates an SSE event representing a RAC delete for an environment. +// version must be greater than the last-known version of that environment. +func MakeAutoConfigDeleteEvent(envID config.EnvironmentID, version int) httphelpers.SSEEvent { + msgBytes, _ := json.Marshal(autoconfig.DeleteMessageData{ + Path: "/environments/" + string(envID), + Version: version, + }) + return httphelpers.SSEEvent{Event: autoconfig.DeleteEvent, Data: string(msgBytes)} +} diff --git a/internal/sharedtest/stream_request.go b/internal/sharedtest/stream_request.go index 2d1939ac..827478ae 100644 --- a/internal/sharedtest/stream_request.go +++ b/internal/sharedtest/stream_request.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "sync" "testing" + "time" "github.com/launchdarkly/eventsource" @@ -84,6 +85,31 @@ func WithStreamRequest( return w.Result() } +// AwaitEventOfType reads from an event channel (such as the one passed to the action by +// WithStreamRequest) until it receives an event whose Event() type matches eventType, skipping any +// events of a different type. It calls t.Fatal if the stream closes (a nil value is received) or +// the timeout elapses before such an event arrives. +func AwaitEventOfType(t *testing.T, eventCh <-chan eventsource.Event, eventType string, timeout time.Duration) eventsource.Event { + t.Helper() + deadline := time.NewTimer(timeout) + defer deadline.Stop() + for { + select { + case e := <-eventCh: + if e == nil { + t.Fatalf("stream closed before receiving event of type %q", eventType) + return nil + } + if e.Event() == eventType { + return e + } + case <-deadline.C: + t.Fatalf("timed out after %s waiting for event of type %q", timeout, eventType) + return nil + } + } +} + func WithStreamRequestLines( t *testing.T, req *http.Request, diff --git a/relay/concurrent_keys_harness_ref_test.go b/relay/concurrent_keys_harness_ref_test.go new file mode 100644 index 00000000..072fcb42 --- /dev/null +++ b/relay/concurrent_keys_harness_ref_test.go @@ -0,0 +1,132 @@ +package relay + +// TestConcurrentKeysHarnessReference is the reference integration test for the concurrent-keys +// test helpers. It demonstrates the reusable helpers added to internal/sharedtest working together +// end-to-end: +// +// - configsource.ArchiveFixtureBuilder (offline-mode archive with flag data) +// - configsource.RACMock (RAC SSE server delivering environment configuration) +// - sharedtest.WithStreamRequest + sharedtest.AwaitEventOfType (consuming Relay's SSE stream) +// +// Feature-specific scenario tests live alongside the code they exercise and reuse these helpers. + +import ( + "net/http" + "testing" + "time" + + c "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/configsource" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + "github.com/launchdarkly/eventsource" + "github.com/launchdarkly/go-configtypes" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + "github.com/launchdarkly/go-server-sdk-evaluation/v3/ldbuilders" + helpers "github.com/launchdarkly/go-test-helpers/v3" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + harnessEnvID = c.EnvironmentID("harness-ref-env-id") + harnessSDKKey = c.SDKKey("sdk-harness-ref-key-001") + harnessMobileKey = c.MobileKey("mob-harness-ref-key-001") + harnessProjKey = "ref-proj" + harnessFlagKey = "harness-simple-flag" +) + +var harnessEnvRep = envfactory.EnvironmentRep{ + EnvID: harnessEnvID, + EnvKey: "harness-ref", + EnvName: "Harness Reference", + ProjKey: harnessProjKey, + ProjName: "Reference Project", + MobKey: harnessMobileKey, + SDKKey: envfactory.SDKKeyRep{Value: harnessSDKKey}, + Version: 1, +} + +// TestConcurrentKeysHarnessReference exercises the reusable concurrent-keys test helpers. +func TestConcurrentKeysHarnessReference(t *testing.T) { + t.Run("archive fixture + SDK stream: flag data flows through Relay's SSE stream", func(t *testing.T) { + // 1. Build an offline-mode archive containing a single env with a simple boolean flag. + archivePath := configsource.NewArchiveFixtureBuilder(). + AddEnv(configsource.ArchiveEnvSpec{ + Rep: harnessEnvRep, + DataID: "data-v1", + Flags: map[string]any{ + harnessFlagKey: ldbuilders.NewFlagBuilder(harnessFlagKey).Version(1).On(true).Build(), + }, + }). + WriteTempFile(t) + + // 2. Start Relay in offline mode with the real archive manager so the flag data actually + // flows through the data store and into the SSE stream. + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientsCreatedCh := make(chan testclient.CapturedLDClient, 4) + cfg := c.Config{} + cfg.OfflineMode.FileDataSource = archivePath + + relay, err := newRelayInternal(cfg, relayInternalOptions{ + loggers: mockLog.Loggers, + clientFactory: testclient.RealLDClientFactoryWithChannel(true, clientsCreatedCh), + // archiveManagerFactory left nil → uses the real filedata.NewArchiveManager + }) + require.NoError(t, err) + defer relay.Close() + + // In offline mode the archive manager loads synchronously, so the client should already + // be in the channel; draining it confirms the environment is ready. + _ = helpers.RequireValue(t, clientsCreatedCh, 3*time.Second, "timed out waiting for SDK client creation") + + // 3. Connect to Relay's server-side SSE stream and verify the initial put event arrives + // and contains the flag. WithStreamRequest drives Relay's handler in-process and cancels + // the request when the action returns, so there is no server/connection teardown to order. + req := sharedtest.BuildRequestWithAuth(http.MethodGet, "/all", harnessSDKKey, nil) + sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { + event := sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) + require.NotNil(t, event) + assert.Contains(t, event.Data(), harnessFlagKey, + "expected put event data to contain the flag key") + }) + }) + + t.Run("RAC mock + SDK stream: Relay discovers env from RAC and serves the SSE stream", func(t *testing.T) { + // 1. Create a RAC mock pre-loaded with a put event for the test environment. + putEvent := configsource.MakeAutoConfigPutEvent(harnessEnvRep) + racMock := configsource.NewRACMock(t, &putEvent) + + // 2. Start Relay configured to use the RAC mock as its config stream. Use CreateDummyClient + // (rather than FakeLDClientFactory) so the data store is initialized with flag data — + // required for Relay to emit a put event on the SSE stream when a client connects. + cfg := c.Config{AutoConfig: c.AutoConfigConfig{Key: testAutoConfKey}} + cfg.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(racMock.URL) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + relay, err := newRelayInternal(cfg, relayInternalOptions{ + loggers: mockLog.Loggers, + clientFactory: testclient.CreateDummyClient, + }) + require.NoError(t, err) + defer relay.Close() + + // 3. Wait for the env to become available (confirms Relay processed the RAC put event). + h := relayTestHelper{t: t, relay: relay} + _ = h.awaitEnvironment(harnessEnvID) + + // 4. Connect to Relay's SSE stream and verify it serves a put event. + req := sharedtest.BuildRequestWithAuth(http.MethodGet, "/all", harnessSDKKey, nil) + sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { + event := sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) + assert.NotNil(t, event, "expected Relay to serve a put event on the SSE stream") + }) + }) +} From 1aa84f40222f136c0379b1b008812685c9da29d6 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 17 Jun 2026 14:34:39 -0700 Subject: [PATCH 06/66] docs(concurrent-keys): incorporate T0 PoC findings and resolve Q5-Q11 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design doc (§7, §9, §13, §14): - §7 rewritten as the concrete T2.c/T2.d specification based on PoC findings. Key change: the in-memory data store is NOT shared across the swap (each client init rebuilds it). The new rule: keep the old store/anchor authoritative until the new client is Initialized(). - §9 adds the reconnect-with-jitter mechanism for malformed payloads — backend has no NAK channel, so we must reconnect to force a fresh put. - §13 (Recorded decisions): removed 'working assumption' qualifiers from Q6/Q7/Q9; added two new decisions (re-anchor store handling and re-anchor atomicity rollback). - §14 (Resolved questions): all of Q5, Q6, Q7, Q8, Q11 now have team-confirmed answers. Plan doc (T1.b, T1.c, T2.c, T2.d, T3.b, T3.c): - T1.b: ReconcileCredentials signals malformed payload via structured error; caller handles. - T1.c: scope narrows — Q8 confirms per-credential downstream tracking already exists. - T2.c: replaces 'TBD per PoC' with concrete order-of-ops, rollback, and store-handling spec. - T2.d: drops httpconfig (no change needed per H4); focuses on big-segment sync re-wire (recreate or replace-credential). - T3.b: helper signals malformed payload as structured error. - T3.c: RAC handler triggers RAC reconnect-with-jitter on malformed payload; offline handler preserves state only. --- docs/concurrent-keys/phase1-design.md | 100 ++++++++++++++++---------- docs/concurrent-keys/phase1-plan.md | 48 ++++++++----- 2 files changed, 96 insertions(+), 52 deletions(-) diff --git a/docs/concurrent-keys/phase1-design.md b/docs/concurrent-keys/phase1-design.md index 0b46e4c5..02b12e23 100644 --- a/docs/concurrent-keys/phase1-design.md +++ b/docs/concurrent-keys/phase1-design.md @@ -238,34 +238,59 @@ On default rotation the backend mirrors expiry info into both: When `sdkKey.value` changes (voluntary rotation *or* current default expiring and being replaced by a promoted non-expiring key), relay must swap its upstream client to the new anchor while preserving downstream SDK connections. -This is the highest-risk piece of Phase 1. The source plan describes the swap in a single sentence ("stand up the new client, hand the data source over, retire the old"). The actual mechanism is *mostly implicit* in today's code rather than orchestrated: +This is the highest-risk piece of Phase 1. The **T0 PoC** validated the swap mechanism against seven hypotheses; the durable tests live in `internal/relayenv/env_context_reanchor_test.go` and the per-hypothesis findings are in [`phase1-T0-reanchor-poc-findings.md`](./phase1-T0-reanchor-poc-findings.md). The findings are summarized below as the spec for T2.c / T2.d. -- The data store is shared via `storeAdapter`, so two SDK clients pointed at the same env can feed the same store as a side-effect. -- `GetClient()` returns `c.clients[c.keyRotator.SDKKey()]` — so flipping the rotator's anchor swaps which client serves. -- Downstream lookup is on `ScopedCredential`, not on the anchor — so downstream connections route correctly regardless of which anchor is current. +### Required order of operations -But several components are wired at **construction time** to the original SDK key and are *not* re-wired by the implicit handoff: +``` +1. Build the new anchor's SDK client (do not flip the anchor pointer yet). +2. Wait for the new client to report Initialized() == true. +3. Atomically flip the rotator's anchor pointer. +4. Call ReplaceCredential on event dispatcher + metrics publisher. +5. Re-wire (or recreate) big-segment sync. +6. Close the old anchor's client (after its grace period elapses for downstream traffic). +``` + +This order is *necessary* — the PoC found that flipping the pointer too early leaves `GetClient()` nil mid-swap (H6) and breaks the env on init failure (H7) — and *sufficient* only with the store-handling caveat below. + +### The data store: old stays authoritative until new is `Initialized()` + +An earlier version of this design assumed two SDK clients pointed at the same env would feed the *same* data store as a side-effect. The PoC (H1, H5) showed this is **wrong for the in-memory store**: each SDK client construction calls `storeAdapter.Build()`, which atomically swaps in a *new, empty* store. The new client must re-sync from scratch. -| Component | Today's wiring | Re-anchor story | +This affects only the in-memory case. With a persistent store (Redis, DynamoDB), the data lives outside the wrapper and survives the swap. + +Three remedies were considered (PoC findings §H5): +1. **Keep the old store/anchor authoritative until the new client is `Initialized()`.** ← **chosen** +2. Require a persistent store for graceful re-anchor. +3. Decouple the data store lifecycle from the client lifecycle. + +Option 1 aligns with §8's "superset during transition" principle: just as the *accepted credential set* is a superset during a keys-change event, the *store* (and the client serving it) stays in place until the replacement is ready. + +### Component re-wiring on re-anchor + +| Component | Today | On re-anchor | |---|---|---| -| Event dispatcher | Stores `authKey`; has `ReplaceCredential` | Call `ReplaceCredential` on re-anchor | -| Metrics publisher | Stores `authKey`; has `ReplaceCredential` | Call `ReplaceCredential` on re-anchor | -| Big-segment sync | Wired to SDK key at construction | **No re-wire path today — new mechanism needed** | -| `httpconfig` | Built with SDK key at construction | Likely key-independent (TLS / proxy config); verify in PoC | +| Event dispatcher | Has `ReplaceCredential` | Call `ReplaceCredential` (T2.c) | +| Metrics publisher | Has `ReplaceCredential` | Call `ReplaceCredential` (T2.c) | +| Big-segment sync | Wired at construction; no replacement method | Re-wire (add a replace-credential method) **or** recreate the synchronizer (T2.d). Recreate is simpler; re-wire preserves in-flight sync state. | +| `httpconfig` | Built via injected *builder* (not pre-built config) | **No change needed** — the SDK rebuilds with the new anchor key automatically (PoC H4) | +| Downstream SSE connections | Keyed on `ScopedCredential`, independent of anchor | Survive automatically (PoC H2); expect one duplicate `put` from the new client's initial sync | -### PoC first +### Failure handling -The re-anchor mechanism is the topic of **T0** — a PoC that validates the swap with concrete tests *before* T2 implements it. The PoC answers seven hypotheses: +If the new client fails to initialize, the swap **rolls back**: the rotator's anchor pointer stays on the old key, the previous accepted set is preserved, a structured error is logged, and an alarm is raised. The old anchor's client (still alive in its grace period) continues to serve. This is the §8 atomicity principle applied to re-anchor. -1. Two clients sharing a store don't corrupt store invariants. -2. Downstream SSE connections tolerate the swap. -3. Big-segment sync keeps working after re-anchor — or, if not, what re-wiring is needed. -4. `httpconfig` stays functional after re-anchor. -5. Order of operations: start-new → swap pointer → close-old vs. alternatives. -6. Behavior during the swap window (requests arriving mid-swap). -7. Failure modes: new client init fails — recovery behavior. +### Consolidated specification for T2.c / T2.d -T0's deliverable is durable test code that survives into T2. +| # | Requirement | Source | Owner | +|---|---|---|---| +| 1 | Build + initialize the new anchor client *before* flipping the pointer; flip atomically. | H5, H6 | T2.c | +| 2 | On init failure, roll back to old anchor; preserve previous accepted set; log + alarm. | H7 | T2.c | +| 3 | Keep the old store/anchor authoritative until new client `Initialized()`. | H1, H5 | T2.c | +| 4 | Re-wire big-segment sync on re-anchor (recreate or replace-credential). | H3 | T2.d | +| 5 | Call `ReplaceCredential` on event dispatcher + metrics publisher. | §7 | T2.c (already wired in `addCredential`) | +| 6 | Expect duplicate downstream `put`; retain connections for credentials still in the accepted set. | H2 | T2.c (awareness) | +| 7 | No `httpconfig` change. | H4 | n/a | --- @@ -305,13 +330,14 @@ Reconcile is **all-or-nothing**. On partial failure (malformed payload, new-clie When relay receives a malformed RAC payload — most importantly, `sdkKey.value` not present in `sdkKeys[]`, or `sdkKey` field missing entirely — the backend invariants of §4.2 have been violated. -**Working assumption** (pending team confirmation): +**Decision** (confirmed with the team): -- **Hard-fail the update.** Log a structured error. Preserve the previous accepted set. Alarm. -- Do *not* silently fall back to the first entry in `sdkKeys[]` (silent and dangerous). -- Do *not* leave the env in a half-applied state. +1. **Preserve the previous accepted set.** Do not apply the malformed update. Log a structured error. Alarm. +2. **Disconnect and reconnect the RAC stream with jitter.** The backend believes the patch was applied — RAC is one-way push and relay has no NAK channel. Without a reconnect the backend won't send a fresh state; it expects relay to be in sync. Reconnecting forces a fresh `put` on the new connection, which gives relay a clean baseline. +3. Do *not* silently fall back to the first entry in `sdkKeys[]` (silent and dangerous). +4. Do *not* leave the env in a half-applied state. -This is the same atomicity principle as §8, applied at the boundary between trusted-source input and relay's internal state. +This is the same atomicity principle as §8, applied at the boundary between trusted-source input and relay's internal state, with the added piece (reconnect) needed because RAC has no acknowledgment mechanism for failed-payload rejection. --- @@ -371,27 +397,29 @@ Environment | Single upstream connection per env on the anchor | Connection-count efficiency at scale; aligns with Phase 2's single-mega-stream model | Multi-client (SDK-2415 PoC approach): trades re-anchor complexity for fan-out at customer scale | | Anchor by `sdkKey.value` byte-match (no `isDefault` flag) | Single source of truth; matches what RAC already emits | `isDefault` flag (would require backend wire change and dual sources of truth) | | Per-key `expiry` (Unix-ms) on array entries | Confirmed real format from producers; reuses existing ticker | Per-env single deprecated slot (today's model — doesn't scale to multi-key) | -| Trust the array on expiry disagreement (Q7 working assumption) | Simpler invariant; legacy field becomes write-only shim | Take whichever is later, hard-fail on disagreement (more complex, no clear value) | +| Trust the array on expiry disagreement | Simpler invariant; legacy field becomes write-only shim | Take whichever is later, hard-fail on disagreement (more complex, no clear value) | | Events collapse to anchor per kind, no per-key attribution | Keys are secrets — not appropriate as analytics tags; LD provides better tagging mechanisms | Per-key attribution (would multiply event machinery N×) | | Diagnostic events keep verbatim-proxy behavior | Preserves operational debug value (which SDK reported); minimal code change | Collapse diagnostic to anchor (loses debug signal); metadata-header (long-term direction, out of Phase 1 scope) | | `ReconcileCredentials` API replaces `UpdateCredential` everywhere | Atomic semantics; single API surface; no external consumers to preserve | Keep both methods (two ways to do the same thing); stateful batching (non-idiomatic Go) | -| Hard-fail on malformed payload (Q6 working assumption) | Loud, safe, atomic | Soft-fall-back to `sdkKeys[0]` (silent, order-dependent); refuse to serve until next valid update (disruptive) | -| Order of operations: add → re-anchor → remove (Q9 working assumption) | Accepted set is a superset during transition; downstream survives | Remove first (downstream-availability window); concurrent (race-prone); atomic batch (atomicity breaks at goroutine boundary) | +| On malformed payload: preserve previous state **+ reconnect RAC stream with jitter** | Loud, safe, atomic — and forces backend to push a fresh `put`, since RAC has no NAK | Soft-fall-back to `sdkKeys[0]` (silent, order-dependent); refuse to serve until next valid update (disruptive); preserve-without-reconnect (backend stays out of sync until something else triggers a refresh) | +| Order of operations: add → re-anchor → remove | Accepted set is a superset during transition; downstream survives | Remove first (downstream-availability window); concurrent (race-prone); atomic batch (atomicity breaks at goroutine boundary) | +| Re-anchor: keep old store/anchor authoritative until new client `Initialized()` | The in-memory store is rebuilt empty on new-client construction (PoC H1, H5); must keep old serving until new is ready | Require persistent store (limits feature to a subset); decouple store from client lifecycle (much bigger refactor) | +| Re-anchor: validate new client `Initialized()` before flipping the anchor pointer; rollback on failure | Avoids mid-swap nil `GetClient()` (PoC H6) and stranded-anchor on init failure (PoC H7) | Flip-then-init (today's broken behavior); accept the gap (visible to customers) | | Manual config stays single-key in Phase 1 | Same trusted-source reasoning as above | Verify-on-startup, opt-in unsafe flag (rejected for the same reasons in §1) | --- -## 14. Open questions (pending offline confirmation) +## 14. Resolved questions -These have working assumptions; Aaron is confirming with the team before lock-in. None block design or initial development. +All design-blocking questions have been answered. -- **Q5**: RAC propagation SLA for `sdkKey.value` changes. *Working assumption*: real-time via SSE push. -- **Q6**: Behavior on malformed RAC payload. *Working assumption*: hard-fail, preserve previous state. -- **Q7**: Legacy `sdkKey.expiring{}` vs per-key `expiry` disagreement policy. *Working assumption*: trust the array. -- **Q8**: Does relay track per-credential downstream connections for targeted disconnect? *Working assumption*: yes (in `envStreams`); verify in code as T1.c pre-work. -- **Q11**: Customer downgrade story (rolling relay back from Phase 1). *Working assumption*: surface in release notes; no relay-side mitigation needed. +- **Q5** (RAC propagation SLA for `sdkKey.value` changes): **Real-time.** Same delivery semantics as flag eval / delivery in the SDK. +- **Q6** (Behavior on malformed RAC payload): **Preserve previous accepted set + reconnect the RAC stream with jitter** to force a fresh `put` from the backend (the backend believes the patch was applied because RAC has no NAK channel). See §9. +- **Q7** (Legacy `sdkKey.expiring{}` vs per-key `expiry` disagreement policy): **Trust the array.** Legacy field is a write-only back-compat shim; new relays ignore it on read. +- **Q8** (Per-credential downstream tracking for targeted disconnect): **Already implemented** — today's rotation/disconnect path uses it. T1.c builds on the existing tracking; does not have to construct new infrastructure. +- **Q11** (Customer downgrade story): **No mitigation work.** Documented in release notes; customers reverting from Phase 1 understand they lose multi-key support. -See [`phase1-questions.md`](../../docs/agents/phase1-questions.md) in the design worktree for full context per question. (That file is not on this feature branch — it lives in the design worktree.) +T0 PoC findings (re-anchoring mechanics) are recorded in §7 and in [`phase1-T0-reanchor-poc-findings.md`](./phase1-T0-reanchor-poc-findings.md). --- diff --git a/docs/concurrent-keys/phase1-plan.md b/docs/concurrent-keys/phase1-plan.md index 0bad9563..7264829c 100644 --- a/docs/concurrent-keys/phase1-plan.md +++ b/docs/concurrent-keys/phase1-plan.md @@ -170,9 +170,13 @@ ReconcileCredentials(newSet AcceptedSet, anchor credential.SDKCredential) error `AcceptedSet` carries the full new state (server keys + mobile keys with optional per-key expiry). The implementation owns the order of operations (`add → re-anchor → remove`) internally; callers don't sequence. +**On malformed payload**: `ReconcileCredentials` should signal the malformed condition (return a structured error) so that the caller can both (a) preserve the previous accepted set and (b) trigger a reconnect of the RAC stream with jitter (per design §9). T1.b owns the API contract; T3.b/c own driving the reconnect. + ### T1.c — Cleanup ticker -Generalize `cleanupExpiredCredentials` (called from `StepTime`) to walk the entire accepted set per kind and drop entries whose `expiry` has passed. The downstream-disconnect logic must handle mobile-key disconnects, not just SDK-key ones. **Q8 pre-work**: verify in code that `envStreams` (or an adjacent component) maintains per-credential downstream connection lists. If not, the per-key targeted disconnect needs additional infrastructure — scope expansion. +Generalize `cleanupExpiredCredentials` (called from `StepTime`) to walk the entire accepted set per kind and drop entries whose `expiry` has passed. The downstream-disconnect logic must handle mobile-key disconnects, not just SDK-key ones. + +**Q8 confirmed by team**: per-credential downstream tracking is *already implemented* — today's rotation/disconnect path uses it. T1.c builds on the existing tracking. No new infrastructure to construct; scope is *narrower* than originally feared. ### T2.a — `addCredential` anchor-only client @@ -184,25 +188,34 @@ The switch case at `env_context_impl.go:448-463` currently calls `startSDKClient ### T2.c — Re-anchor mechanism -The big one. Implements whatever order-of-ops and component-rewiring the PoC settled on. Per the §7 design analysis, the swap consists of: +The big one. PoC findings (design §7 + [`phase1-T0-reanchor-poc-findings.md`](./phase1-T0-reanchor-poc-findings.md)) turned this from "TBD per PoC" into a concrete specification: + +1. **Build** the new anchor's SDK client (do *not* flip the anchor pointer yet). +2. **Wait** for the new client to report `Initialized() == true`. +3. **Atomically flip** the rotator's anchor pointer. Until this moment, `GetClient()` returns the **old** client and evaluations are served from the old store. +4. **Call `ReplaceCredential`** on the event dispatcher and metrics publisher. +5. **Re-wire** (or recreate) big-segment sync — T2.d owns this piece; T2.c calls into it. +6. **Close** the old upstream client after its grace period elapses for any retained downstream traffic. + +**On new-client init failure**: roll back. Anchor pointer stays on the old key, previous accepted set is preserved, structured error logged, alarm raised. The old client (still alive in its grace period) continues to serve. -1. Start new upstream client on the new anchor's SDK key. -2. Wait for it to initialize (the data store is shared; the new client feeds the same store). -3. Atomically swap the rotator's anchor pointer (so `GetClient()` returns the new client). -4. Call `ReplaceCredential` on event dispatcher and metrics publisher. -5. Re-wire big-segment sync (mechanism TBD per PoC). -6. Verify `httpconfig` continues to work (likely no-op per PoC). -7. Close the old upstream client. +**Why this order matters** (PoC H1, H5, H6, H7): +- Flipping the pointer before the new client is registered leaves `GetClient()` returning nil mid-swap (H6). +- Flipping the pointer on init failure strands the env with no usable client even though the old one is fine (H7). +- The in-memory store is rebuilt empty when the new client constructs (H1, H5) — keeping the old anchor authoritative until `Initialized()` is the only way to avoid an evaluation gap without requiring a persistent store. -PoC failure modes inform the recovery logic. +**Awareness for T2.c**: downstream SSE connections survive the swap automatically but will receive *one duplicate `put`* from the new client's initial sync (PoC H2). This is tolerable and expected — don't treat it as a bug. -### T2.d — Big-segment sync + `httpconfig` from anchor +### T2.d — Big-segment sync re-wire on re-anchor -These are the two construction-time wirings. Either: -- Refactor big-segment sync to be re-wireable (add a method to point it at a new SDK key), or -- Recreate the big-segment sync component on each re-anchor (heavier but simpler). +T2.d's single responsibility (after PoC): re-wire big-segment sync when the anchor changes. Choose one approach at PR time: -`httpconfig` is mostly TLS / proxy / event-base-uri config — key-independent — but verify in PoC. +- **Recreate**: destroy and reconstruct the `BigSegmentSynchronizer` on each re-anchor. Simpler; loses any in-flight sync state. +- **Replace-credential**: add a method to the synchronizer interface that updates its SDK key in place. Preserves in-flight state; requires a new method on the interface. + +**Recommendation**: recreate, unless we discover in-flight state preservation matters for a specific big-segment customer scenario. Recreate is the easier path; switch to replace-credential only if needed. + +`httpconfig` was previously scoped to this task — **PoC H4 confirmed no `httpconfig` change is needed**. The SDK rebuilds the HTTP config with the new anchor key automatically because relay injects the *builder*, not a pre-built config. Removed from T2.d's scope. ### T2.e — Handler fan-out optimization @@ -220,13 +233,15 @@ A new helper (in `internal/envfactory/` or similar) that both `autoconfig_action - Diff the old accepted set against the new one (set-keyed by `value`). - Detect re-anchor (`sdkKey.value` changed). - Compute the ordered operation list: `add → re-anchor → remove`. -- Hard-fail if the payload is malformed (anchor `value` not in `sdkKeys[]`). +- **Signal malformed-payload condition** (anchor `value` not in `sdkKeys[]`) as a structured error so the caller can both preserve the previous state *and* trigger an RAC stream reconnect with jitter (design §9). - Treat the legacy `sdkKey.expiring{}` field as write-only — read only the array. ### T3.c — Wire both action handlers Replace `UpdateCredential` calls with the new `ReconcileCredentials` API, via the shared helper. RAC handler and offline handler updates land in one PR (separate commits per Aaron's preference). +**Malformed-payload handling** (design §9): when the shared helper signals a malformed payload, the RAC handler must (a) preserve the previous accepted set and (b) **disconnect and reconnect the RAC stream with jitter** to force a fresh `put` from the backend. The offline handler preserves state only (no equivalent reconnect since there's no live connection — wait for the next archive reload). + Test matrix (covered in T3.c's acceptance criteria): - Add a new key - Set `expiry` on a non-anchor key @@ -236,6 +251,7 @@ Test matrix (covered in T3.c's acceptance criteria): - De-expiry (remove `expiry` on existing entry — cancel scheduled drop) - Mixed patch (add + re-anchor + remove) - Partial-failure reconcile (preserves previous state) +- **Malformed payload triggers RAC reconnect** (RAC handler only); state preserved meanwhile ### T4 — Status endpoint arrays From b0214ce38c036cd6802199b19e1170f6f11b6ebb Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 17 Jun 2026 14:54:29 -0700 Subject: [PATCH 07/66] docs(concurrent-keys): add T5.f (cleanup) and T5.g (merge+release) to Wave 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit T5 scope renamed to 'Tests, release, and merge-forward'. The Wave 3 chain becomes [Wave 2 terminal nodes] → T5.f (remove project scaffolding) → T5.g (squash-merge to v8 + release) → T5.e (merge-forward to v9, calendar-deferred). T5.f removes docs/concurrent-keys/ and the PoC test file before merge; this file is part of what gets removed by T5.f itself. T5.g squash-merges feat/concurrent-keys to v8 as a single feat: commit, triggering the minor version bump. JIRA tickets created: SDK-2555 (T5.f) and SDK-2556 (T5.g), both under SDK-2535. Dependencies wired. --- docs/concurrent-keys/phase1-plan.md | 49 +++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/concurrent-keys/phase1-plan.md b/docs/concurrent-keys/phase1-plan.md index 7264829c..4b52aec2 100644 --- a/docs/concurrent-keys/phase1-plan.md +++ b/docs/concurrent-keys/phase1-plan.md @@ -28,13 +28,13 @@ This document is committed in that worktree at `docs/concurrent-keys/phase1-plan ## 2. Wave breakdown -Three waves. Wave 3 is the final merge-forward to v9, possibly weeks or months after Wave 2 completes. +Three waves. Wave 3 is release-time work: clean up project scaffolding, merge `feat/concurrent-keys` to v8 and release, then merge-forward to v9 when v9 is ready (possibly weeks or months later). | Wave | Theme | When | |---|---|---| | **Wave 1** | Foundations: PoC, data structures, wire types, test infrastructure | Immediately, multiple sub-tasks in parallel | | **Wave 2** | Core implementation: API surface change, re-anchor mechanism, peripheral re-wiring, end-to-end integration | After PoC findings + Wave 1 data structures land | -| **Wave 3** | Merge-forward to v9 | Possibly weeks/months after Wave 2, depending on v8 production deploy timing | +| **Wave 3** | Release: code cleanup → merge to v8 → publish release → merge-forward to v9 | Cleanup + v8 release happen as soon as Wave 2 completes; merge-forward to v9 is calendar-deferred | The "single-key behavior unchanged at every PR boundary" invariant is the load-bearing testable property. Every sub-PR must preserve it. @@ -78,9 +78,11 @@ Each task has: ticket name, files touched, dependencies, estimates. Acceptance c | Task | Files | Depends on | Human | AI agent | |---|---|---|---|---| -| **T5.e** — Merge-forward to v9 | `internal/relayenv/*`, streaming path | All Wave 2 done | 3-7 days | 1-2 days (with iteration) | +| **T5.f** — Code cleanup: remove project scaffolding before v8 merge | `docs/concurrent-keys/` (entire directory), `internal/relayenv/env_context_reanchor_test.go`, anything else project-specific | All Wave 2 terminal sub-tasks (T1.c, T2.b, T2.d, T2.e, T4) | 0.5-1 day | 30 min - 1 hr | +| **T5.g** — Merge `feat/concurrent-keys` to v8 + publish release | None (release activity) | T5.f | 0.5-1 day | n/a (release task, not coding) | +| **T5.e** — Merge-forward to v9 | `internal/relayenv/*`, streaming path | T5.g (and calendar — may be weeks/months after the v8 release) | 3-7 days | 1-2 days (with iteration) | -**Total project**: ~6-11 weeks of full-time human work (excluding the calendar gap before Wave 3, which may extend the project's wall-clock duration substantially). +**Total project**: ~6-11 weeks of full-time human work for code work; the merge-forward to v9 (T5.e) lives on its own calendar that depends on v9 readiness. --- @@ -106,13 +108,13 @@ T3.b ───────────────────────── T5.a (test harness) — supports all other tasks' tests T5.b (events regression) — runs continuously after landing -[all Wave 2 done] ─→ T5.e (merge-forward to v9) +[Wave 2 terminal nodes: T1.c, T2.b, T2.d, T2.e, T4] ─→ T5.f (cleanup) ─→ T5.g (merge to v8 + release) ─→ T5.e (merge-forward to v9) ``` **Critical path** (longest dependency chain): -T1.0 → T1.a → T1.b → T2.a → T2.c → T2.e → T5.e +T1.0 → T1.a → T1.b → T2.a → T2.c → T2.e → T5.f → T5.g → T5.e -This chain alone is roughly: 0.5 + 1.5 + 2.5 + 1.5 + 4 + 2.5 + 5 = ~17 human days at the midpoint of the estimates. Other Wave 2 tasks parallelize off this critical path. +The Wave 2 portion is roughly: 0.5 + 1.5 + 2.5 + 1.5 + 4 + 2.5 = ~12.5 human days at the midpoint. Wave 3 adds ~1 day (cleanup) + ~1 day (merge/release) + 3-7 days (v9 merge-forward, calendar-deferred). Other Wave 2 tasks parallelize off this critical path. --- @@ -274,11 +276,38 @@ The harness lands as Wave 1 infrastructure; scenarios accumulate as acceptance t Capture upstream payloads from v8 under realistic SDK traffic. Assert post-Phase-1 payloads are structurally identical *except* for the credential field. Catches accidental schema drift throughout the project. +### T5.f — Code cleanup before v8 merge + +Remove all project-specific scaffolding from `feat/concurrent-keys` *before* T5.g merges the branch to v8. The canonical design + plan docs and the PoC test file were useful during development; they shouldn't land on v8. + +What to remove: +- `docs/concurrent-keys/` — entire directory (this file is one of the things being removed). Save off-branch if you want to keep it for reference. +- `internal/relayenv/env_context_reanchor_test.go` — PoC test file. Verify any useful tests have already been adopted into proper regression test files by T2.c before deleting. +- Any other concurrent-keys-specific scaffolding that may have accumulated. + +What stays: +- Actual feature code. +- Regression tests in properly-named test files (those aren't scaffolding). + +Single PR. Conventional commit: `chore(concurrent-keys): remove project scaffolding before v8 merge`. + +### T5.g — Merge `feat/concurrent-keys` to v8 + publish release + +Final merge. **Squash-merge** as a single `feat:` commit — that commit is what release tooling sees, so it triggers the minor version bump. Suggested squash title: `feat(concurrent-keys): support multiple SDK keys per environment via RAC and offline archive`. + +Steps: +1. Final review of feature branch HEAD; confirm cleanup (T5.f) is in. +2. Squash-merge `feat/concurrent-keys` → v8. +3. Verify release tooling triggers a minor version bump. +4. Publish release notes (three items from §8). + +Not a coding task — this is a release activity. + ### T5.e — Merge-forward to v9 Not a `git merge`. Real integration work resolving FDv2 ↔ Phase 1 interactions in `env_context_impl.go` and the streaming path. v9 has FDv2 in it, which touches the same files Phase 1 changes most heavily. Validate against v9's existing test suite plus a subset of Phase 1 tests adapted for v9. -Timing: possibly weeks or months after Wave 2 completes, depending on when v8 ships to production. +Timing: calendar-deferred. May happen weeks or months after T5.g (v8 release), depending on when v9 is ready. --- @@ -403,9 +432,11 @@ SDK-2453 (Epic) — Relay Proxy Multi Keys Support │ ├── T3.b Shared reconcile helper [Sub-task] │ └── T3.c Wire RAC + offline handlers [Sub-task] ├── T4 Status endpoints [Task] -└── T5 Tests + merge-forward [Story] +└── T5 Tests, release, and merge-forward [Story] ├── T5.a Integration test harness [Sub-task] ├── T5.b Events payload regression test [Sub-task] + ├── T5.f Code cleanup before v8 merge [Sub-task] + ├── T5.g Merge feat/concurrent-keys to v8 + release [Sub-task] └── T5.e Merge-forward to v9 [Sub-task] ``` From a0b07ceb5d67c313348581f75fa24dcf6cd02cfc Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Thu, 18 Jun 2026 09:34:17 -0700 Subject: [PATCH 08/66] feat(credential): add Rotator accepted-set data structures (#706) Adds internal accepted-set maps (acceptedSDKKeys, acceptedMobileKeys) to Rotator, each entry holding an optional expiry timestamp (*time.Time, nil = permanent). Populates the maps from Initialize() so a single-key environment immediately reflects the accepted set correctly. --- internal/credential/rotator.go | 13 +++++++++++++ internal/credential/rotator_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 1bc2e62b..536a63fe 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -9,6 +9,11 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" ) +// acceptedKeyInfo holds per-key metadata for the accepted-set maps. +type acceptedKeyInfo struct { + expiry *time.Time //nolint:unused // nil = permanent; read by T1.b (ReconcileCredentials) +} + type Rotator struct { loggers ldlog.Loggers @@ -31,6 +36,10 @@ type Rotator struct { // Upon expiration, they are removed. deprecatedSdkKeys map[config.SDKKey]time.Time + // Consumed by ReconcileCredentials API + acceptedSDKKeys map[config.SDKKey]*acceptedKeyInfo + acceptedMobileKeys map[config.MobileKey]*acceptedKeyInfo + expirations []SDKCredential additions []SDKCredential @@ -50,6 +59,8 @@ func NewRotator(loggers ldlog.Loggers) *Rotator { loggers: loggers, deprecatedSdkKeys: make(map[config.SDKKey]time.Time), deprecatedMobileKeys: make(map[config.MobileKey]time.Time), + acceptedSDKKeys: make(map[config.SDKKey]*acceptedKeyInfo), + acceptedMobileKeys: make(map[config.MobileKey]*acceptedKeyInfo), } return r } @@ -67,8 +78,10 @@ func (r *Rotator) Initialize(credentials []SDKCredential) { switch cred := cred.(type) { case config.SDKKey: r.primarySdkKey = cred + r.acceptedSDKKeys[cred] = &acceptedKeyInfo{} case config.MobileKey: r.primaryMobileKey = cred + r.acceptedMobileKeys[cred] = &acceptedKeyInfo{} case config.EnvironmentID: r.primaryEnvironmentID = cred } diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index d06379de..86c0a346 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -232,6 +232,34 @@ func TestSDKKeyExpiredInThePastIsNotAdded(t *testing.T) { assert.Empty(t, expirations) } +func TestInitializePopulatesAcceptedSets(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + sdkKey := config.SDKKey("sdk-test-key") + mobileKey := config.MobileKey("mob-test-key") + envID := config.EnvironmentID("env-test-id") + + rotator.Initialize([]SDKCredential{sdkKey, mobileKey, envID}) + + // Verify accepted SDK key set: one entry, no expiry. + assert.Len(t, rotator.acceptedSDKKeys, 1) + if info, ok := rotator.acceptedSDKKeys[sdkKey]; assert.True(t, ok, "acceptedSDKKeys should contain the initialized SDK key") { + assert.Nil(t, info.expiry, "a key initialized without expiry should have nil expiry in acceptedKeyInfo") + } + + // Verify accepted mobile key set: one entry, no expiry. + assert.Len(t, rotator.acceptedMobileKeys, 1) + if info, ok := rotator.acceptedMobileKeys[mobileKey]; assert.True(t, ok, "acceptedMobileKeys should contain the initialized mobile key") { + assert.Nil(t, info.expiry, "a key initialized without expiry should have nil expiry in acceptedKeyInfo") + } + + // Existing public API is unchanged. + assert.Equal(t, sdkKey, rotator.SDKKey()) + assert.Equal(t, mobileKey, rotator.MobileKey()) + assert.Equal(t, envID, rotator.EnvironmentID()) +} + func TestRotateWithGraceMobileKey(t *testing.T) { t.Run("does not panic with non-nil grace period", func(t *testing.T) { mockLog := ldlogtest.NewMockLog() From 7f58595817d61690c56257cd8a775cf35af09fd9 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Thu, 18 Jun 2026 09:32:27 -0700 Subject: [PATCH 09/66] fix(server): eliminate send-on-closed-channel race in StartHTTPServer (#708) Fix a data race in StartHTTPServer where the shutdown goroutine calls close(errCh) concurrently with the listener goroutine sending to errCh, producing a "send on closed channel" panic. --- internal/application/server.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/application/server.go b/internal/application/server.go index afcb9570..1cf40fb2 100644 --- a/internal/application/server.go +++ b/internal/application/server.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "os/signal" + "sync" "syscall" "time" @@ -39,13 +40,15 @@ func StartHTTPServer( } } - errCh := make(chan error) + errCh := make(chan error, 1) // Create a channel to listen for signals sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGTERM) - go func() { + var wg sync.WaitGroup + + wg.Go(func() { var err error loggers.Infof("Starting server listening on port %d\n", port) if tlsEnabled { @@ -61,7 +64,7 @@ func StartHTTPServer( if err != nil && err != http.ErrServerClosed { errCh <- err } - }() + }) // Handle graceful shutdown in a separate goroutine go func() { @@ -83,7 +86,8 @@ func StartHTTPServer( } else { loggers.Info("Server gracefully stopped") } - close(errCh) // Close the error channel after shutdown + wg.Wait() + close(errCh) }() return srv, errCh From cc3134bae3875b25096846158f0be281241c3f59 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Thu, 18 Jun 2026 10:41:08 -0700 Subject: [PATCH 10/66] feat(envfactory): extend EnvironmentRep and EnvironmentParams for concurrent keys (#702) - Adds ConcurrentKeyRep wire type for entries in the sdkKeys/mobileKeys arrays - Adds SDKKeys []ConcurrentKeyRep and MobileKeys []ConcurrentKeyRep to EnvironmentRep - Adds AcceptedSDKKey, AcceptedMobileKey, and corresponding slice fields to EnvironmentParams --- internal/envfactory/env_params.go | 27 ++++++++ internal/envfactory/env_rep.go | 93 ++++++++++++++++++++++++++++ internal/envfactory/env_rep_test.go | 95 +++++++++++++++++++++++++++-- 3 files changed, 211 insertions(+), 4 deletions(-) diff --git a/internal/envfactory/env_params.go b/internal/envfactory/env_params.go index d0230c3f..a0d02a0a 100644 --- a/internal/envfactory/env_params.go +++ b/internal/envfactory/env_params.go @@ -30,6 +30,17 @@ type EnvironmentParams struct { // the canonical one). ExpiringSDKKey ExpiringSDKKey + // AcceptedSDKKeys is the full accepted set of SDK keys for this environment, including the + // anchor. Always non-nil after ToParams(): non-empty sdkKeys arrays populate directly; absent + // or empty sdkKeys are synthesized from the singular sdkKey field so there is always at least + // the anchor entry. + AcceptedSDKKeys []AcceptedSDKKey + + // AcceptedMobileKeys is the full accepted set of mobile keys for this environment. Always + // non-nil after ToParams(): non-empty mobileKeys arrays populate directly; absent or empty + // mobileKeys are synthesized from the singular mobKey field. + AcceptedMobileKeys []AcceptedMobileKey + // TTL is the cache TTL for PHP clients. TTL time.Duration @@ -37,6 +48,22 @@ type EnvironmentParams struct { SecureMode bool } +// AcceptedSDKKey is one entry in the accepted SDK key set for an environment. +// Expiry is zero if the key is permanent. +type AcceptedSDKKey struct { + Key string + Value config.SDKKey + Expiry time.Time +} + +// AcceptedMobileKey is one entry in the accepted mobile key set for an environment. +// Expiry is zero if the key is permanent. +type AcceptedMobileKey struct { + Key string + Value config.MobileKey + Expiry time.Time +} + type ExpiringSDKKey struct { Key config.SDKKey Expiration time.Time diff --git a/internal/envfactory/env_rep.go b/internal/envfactory/env_rep.go index 3d46cced..ace2a54f 100644 --- a/internal/envfactory/env_rep.go +++ b/internal/envfactory/env_rep.go @@ -15,6 +15,41 @@ import ( // or the other of those contexts should be in the appropriate package instead of here. // EnvironmentRep is a representation of an environment that is being added or updated. +// +// EnvironmentRep carries an environment's wire shape from RAC and the offline archive +// (same struct serves both — keep them aligned). +// +// FIELD NAMING — read this before changing anything: +// +// sdkKey is the singular *default* SDK key for the environment. It's an +// object ({"value": "sdk-..."}) so it can also carry the legacy +// sdkKey.expiring{value, timestamp} slot during default rotation +// (back-compat for relays predating concurrent keys). +// +// mobKey is the singular default mobile key. It's a *plain string* +// because mobile keys never had a legacy expiring slot. The shape +// asymmetry is historical, not a design choice. +// +// sdkKeys/mobileKeys are the authoritative full accepted set. Entries: +// { key: , value: , expiry?: } +// +// TERMINOLOGY: +// +// The wire "key" field is the human-readable identifier (e.g. "default-sdk"), +// non-secret — stored as AcceptedSDKKey.Key / AcceptedMobileKey.Key internally. +// The wire "value" field is the actual credential string (e.g. "sdk-xxxx-..."), +// which is the secret — stored as AcceptedSDKKey.Value / AcceptedMobileKey.Value. +// Note that relay's own types (SDKKey, MobileKey, SDKCredential) refer to what +// the wire calls "value" — they are misnamed by today's standards but stable, +// so do not rename them. +// +// Anchor selection: anchor = the sdkKeys entry whose `value` matches sdkKey.value. +// No isDefault flag — value match is the signal. +// +// Backwards compatibility: Go's default JSON decoder ignores unknown fields, so old +// relays receiving payloads with sdkKeys/mobileKeys simply ignore them and continue +// using sdkKey/mobKey. DisallowUnknownFields is intentionally not used anywhere in +// this parse path. type EnvironmentRep struct { EnvID config.EnvironmentID `json:"envID"` EnvKey string `json:"envKey"` @@ -23,6 +58,8 @@ type EnvironmentRep struct { ProjKey string `json:"projKey"` ProjName string `json:"projName"` SDKKey SDKKeyRep `json:"sdkKey"` + SDKKeys []ConcurrentKeyRep `json:"sdkKeys,omitempty"` + MobileKeys []ConcurrentKeyRep `json:"mobileKeys,omitempty"` DefaultTTL int `json:"defaultTtl"` SecureMode bool `json:"secureMode"` Version int `json:"version"` @@ -62,6 +99,17 @@ type ExpiringKeyRep struct { Timestamp ldtime.UnixMillisecondTime `json:"timestamp"` } +// ConcurrentKeyRep is an entry in the sdkKeys or mobileKeys array on EnvironmentRep. +// It represents one accepted credential in an environment's concurrent key set. +// +// Key is the human-readable identifier (non-secret, e.g. "default-sdk"); Value is +// the credential secret (e.g. "sdk-xxxx-..."). See the EnvironmentRep TERMINOLOGY comment. +type ConcurrentKeyRep struct { + Key string `json:"key"` + Value string `json:"value"` + Expiry *int64 `json:"expiry,omitempty"` // Unix-ms; nil = permanent +} + func (e ExpiringKeyRep) ToParams() ExpiringSDKKey { if e.Value.Defined() { return ExpiringSDKKey{ @@ -94,6 +142,51 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { SecureMode: r.SecureMode, } + if len(r.SDKKeys) > 0 { + // New-format payload: populate directly from the array. + params.AcceptedSDKKeys = make([]AcceptedSDKKey, 0, len(r.SDKKeys)) + for _, k := range r.SDKKeys { + entry := AcceptedSDKKey{ + Key: k.Key, + Value: config.SDKKey(k.Value), + } + if k.Expiry != nil { + entry.Expiry = time.UnixMilli(*k.Expiry) + } + params.AcceptedSDKKeys = append(params.AcceptedSDKKeys, entry) + } + } else { + // Old-format payload: no sdkKeys array present. Synthesize AcceptedSDKKeys from the + // singular sdkKey fields so consumers always receive a consistent non-nil model + // regardless of wire format version. Key (identifier) is empty — the old format had none. + params.AcceptedSDKKeys = make([]AcceptedSDKKey, 0, 2) + params.AcceptedSDKKeys = append(params.AcceptedSDKKeys, AcceptedSDKKey{Value: r.SDKKey.Value}) + if r.SDKKey.Expiring.Value.Defined() { + params.AcceptedSDKKeys = append(params.AcceptedSDKKeys, AcceptedSDKKey{ + Value: r.SDKKey.Expiring.Value, + Expiry: ToTime(r.SDKKey.Expiring.Timestamp), + }) + } + } + + if len(r.MobileKeys) > 0 { + // New-format payload: populate directly from the array. + params.AcceptedMobileKeys = make([]AcceptedMobileKey, 0, len(r.MobileKeys)) + for _, k := range r.MobileKeys { + entry := AcceptedMobileKey{ + Key: k.Key, + Value: config.MobileKey(k.Value), + } + if k.Expiry != nil { + entry.Expiry = time.UnixMilli(*k.Expiry) + } + params.AcceptedMobileKeys = append(params.AcceptedMobileKeys, entry) + } + } else { + // Old-format payload: synthesize from the singular mobKey field. + params.AcceptedMobileKeys = []AcceptedMobileKey{{Value: r.MobKey}} + } + return params } diff --git a/internal/envfactory/env_rep_test.go b/internal/envfactory/env_rep_test.go index b46e7566..1d48e237 100644 --- a/internal/envfactory/env_rep_test.go +++ b/internal/envfactory/env_rep_test.go @@ -35,10 +35,12 @@ func TestEnvironmentRepToParams(t *testing.T) { ProjKey: "projkey1", ProjName: "projname1", }, - SDKKey: env1.SDKKey.Value, - MobileKey: env1.MobKey, - TTL: 2 * time.Minute, - SecureMode: true, + SDKKey: env1.SDKKey.Value, + MobileKey: env1.MobKey, + TTL: 2 * time.Minute, + SecureMode: true, + AcceptedSDKKeys: []AcceptedSDKKey{{Value: env1.SDKKey.Value}}, + AcceptedMobileKeys: []AcceptedMobileKey{{Value: env1.MobKey}}, }, params1) env2 := EnvironmentRep{ @@ -70,6 +72,11 @@ func TestEnvironmentRepToParams(t *testing.T) { Expiration: time.UnixMilli(int64(env2.SDKKey.Expiring.Timestamp)), }, MobileKey: env2.MobKey, + AcceptedSDKKeys: []AcceptedSDKKey{ + {Value: env2.SDKKey.Value}, + {Value: env2.SDKKey.Expiring.Value, Expiry: time.UnixMilli(int64(env2.SDKKey.Expiring.Timestamp))}, + }, + AcceptedMobileKeys: []AcceptedMobileKey{{Value: env2.MobKey}}, }, params2) } @@ -105,3 +112,83 @@ func TestEnvironmentRepJSONFormat(t *testing.T) { SecureMode: true, }, rep) } + +// TestEnvironmentRepNewFormatWithArrays parses a realistic RAC put payload carrying the new +// sdkKeys/mobileKeys arrays and verifies the struct fields are populated correctly. +func TestEnvironmentRepNewFormatWithArrays(t *testing.T) { + expiryMs := int64(1700000000000) + jsonStr := `{ + "envID": "68e5179e8307e4099c277e2a", + "envKey": "production", + "envName": "Production", + "mobKey": "mob-f41c", + "projKey": "my-project", + "projName": "My Project", + "sdkKey": { "value": "sdk-anchor" }, + "sdkKeys": [ + { "key": "default-sdk", "value": "sdk-anchor" }, + { "key": "service-a", "value": "sdk-service-a", "expiry": 1700000000000 } + ], + "mobileKeys": [ + { "key": "mob-key-1", "value": "mob-f41c" } + ], + "secureMode": false, + "version": 26 + }` + + var rep EnvironmentRep + require.NoError(t, json.Unmarshal([]byte(jsonStr), &rep)) + + assert.Equal(t, config.SDKKey("sdk-anchor"), rep.SDKKey.Value) + assert.Equal(t, config.MobileKey("mob-f41c"), rep.MobKey) + + require.Len(t, rep.SDKKeys, 2) + assert.Equal(t, ConcurrentKeyRep{Key: "default-sdk", Value: "sdk-anchor"}, rep.SDKKeys[0]) + assert.Equal(t, ConcurrentKeyRep{Key: "service-a", Value: "sdk-service-a", Expiry: &expiryMs}, rep.SDKKeys[1]) + + require.Len(t, rep.MobileKeys, 1) + assert.Equal(t, ConcurrentKeyRep{Key: "mob-key-1", Value: "mob-f41c"}, rep.MobileKeys[0]) + + params := rep.ToParams() + + require.Len(t, params.AcceptedSDKKeys, 2) + assert.Equal(t, AcceptedSDKKey{Key: "default-sdk", Value: config.SDKKey("sdk-anchor")}, params.AcceptedSDKKeys[0]) + assert.Equal(t, AcceptedSDKKey{ + Key: "service-a", + Value: config.SDKKey("sdk-service-a"), + Expiry: time.UnixMilli(expiryMs), + }, params.AcceptedSDKKeys[1]) + + require.Len(t, params.AcceptedMobileKeys, 1) + assert.Equal(t, AcceptedMobileKey{Key: "mob-key-1", Value: config.MobileKey("mob-f41c")}, params.AcceptedMobileKeys[0]) +} + +// TestEnvironmentRepOldFormatNoArrays verifies that an old-format payload (singular sdkKey/mobKey +// only, no sdkKeys/mobileKeys arrays) is normalized by ToParams() into a consistent accepted set. +// The wire rep's SDKKeys/MobileKeys remain nil, but params.AcceptedSDKKeys/AcceptedMobileKeys are +// synthesized from the singular fields so consumers never need to handle two code paths. +func TestEnvironmentRepOldFormatNoArrays(t *testing.T) { + jsonStr := `{ + "envID": "envid1", + "envKey": "envkey", + "envName": "envname", + "mobKey": "mob-default", + "projKey": "projkey", + "projName": "projname", + "sdkKey": { "value": "sdk-key1" }, + "secureMode": false + }` + + var rep EnvironmentRep + require.NoError(t, json.Unmarshal([]byte(jsonStr), &rep)) + assert.Nil(t, rep.SDKKeys) + assert.Nil(t, rep.MobileKeys) + + params := rep.ToParams() + assert.Equal(t, config.SDKKey("sdk-key1"), params.SDKKey) + assert.Equal(t, config.MobileKey("mob-default"), params.MobileKey) + require.Len(t, params.AcceptedSDKKeys, 1) + assert.Equal(t, AcceptedSDKKey{Value: config.SDKKey("sdk-key1")}, params.AcceptedSDKKeys[0]) + require.Len(t, params.AcceptedMobileKeys, 1) + assert.Equal(t, AcceptedMobileKey{Value: config.MobileKey("mob-default")}, params.AcceptedMobileKeys[0]) +} From c8b4b6ef6b78e5c81e7d069da0a05d2fb3a442af Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Thu, 18 Jun 2026 14:03:31 -0700 Subject: [PATCH 11/66] test(relayenv): re-anchoring PoC tests and findings (#704) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds durable TestReanchorPoC_H* tests in env_context_reanchor_test.go that exercise today's UpdateCredential grace-period rotation and document seven hypotheses (store rebuild, SSE survival, big-segment re-wire, ordering, failure rollback, etc.) as the executable spec for T2.c. Adds .agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md and updates phase1-design.md §7 with validated behavior—especially store handover (SSERelayDataStoreAdapter reusing the existing store) instead of an empty in-memory window after swap. --- .../phase1-T0-reanchor-poc-findings.md | 182 +++++ .../concurrent-keys/phase1-design.md | 22 +- .../concurrent-keys/phase1-plan.md | 10 +- .../relayenv/env_context_reanchor_test.go | 622 ++++++++++++++++++ 4 files changed, 819 insertions(+), 17 deletions(-) create mode 100644 .agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md rename {docs => .agent-docs}/concurrent-keys/phase1-design.md (92%) rename {docs => .agent-docs}/concurrent-keys/phase1-plan.md (97%) create mode 100644 internal/relayenv/env_context_reanchor_test.go diff --git a/.agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md b/.agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md new file mode 100644 index 00000000..e27c5e6f --- /dev/null +++ b/.agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md @@ -0,0 +1,182 @@ +# T0 — Re-anchoring PoC: Findings + +**Ticket**: [SDK-2530](https://launchdarkly.atlassian.net/browse/SDK-2530) +**Epic**: [SDK-2453](https://launchdarkly.atlassian.net/browse/SDK-2453) +**Design**: [`phase1-design.md`](./phase1-design.md) §7 "Re-anchoring" +**Tests**: [`internal/relayenv/env_context_reanchor_test.go`](../../internal/relayenv/env_context_reanchor_test.go) + +## Purpose + +Validate the upstream SDK-client swap mechanism that **T2.c** will implement, answering the seven +hypotheses from design §7 with durable tests *before* T2 begins. T0 validates feasibility; T2.c +implements. The tests in `env_context_reanchor_test.go` are written against today's primitives so they +survive into T2 as regression tests and as the executable spec for the swap. + +There is no dedicated re-anchor method yet. The closest existing code path is `UpdateCredential` with a +grace period (rotate the primary SDK key, stand up a new client, keep the old one alive during a grace +window). Several PoC tests drive that path and observe where it falls short of the §7 requirements; +those gaps are the concrete acceptance criteria for T2.c. + +## Headline conclusion + +**Re-anchoring is feasible, but it is _not_ a transparent side-effect of today's code — three concrete +gaps must be closed by T2.c/T2.d, and one design assumption (the "shared store") is only true for +persistent stores.** None of the gaps are blockers; each has a clear remedy. The single highest-risk +item is the in-memory data store being rebuilt (emptied) when the new anchor client starts; the remedy is +to hand the existing store over to the new client rather than rebuild it (H5). + +--- + +## Findings per hypothesis + +### H1 — Two SDK clients sharing a `storeAdapter` don't corrupt store invariants + +**Answer: No corruption, but the "shared store" is a misconception for the in-memory store.** + +`SSERelayDataStoreAdapter.Build()` is called once per SDK-client creation (the SDK invokes +`DataStore.Build()` during client init). Each call constructs a **new** `streamUpdatesStoreWrapper` +around a **freshly built** underlying store and atomically swaps `adapter.store` to point at it. So: + +- With the **default in-memory** store, the second client (the new anchor) gets a brand-new, empty, + uninitialized store. Design §7's "two SDK clients … can feed the same store as a side-effect" does + **not** hold here — the new client must re-sync from scratch. +- With a **persistent** store (Redis/DynamoDB), `wrappedFactory.Build()` returns a handle to the same + external database, so the data (and `IsInitialized()`) survive the swap. This is the only + configuration in which the §7 assumption is literally true. + +No invariant corruption occurs in either case (the swap is atomic under the adapter's lock), but the +emptiness of the new in-memory store is the crux of H5. The remedy — handing the existing store over to +the new client rather than rebuilding — is covered under H5. + +### H2 — Downstream SSE connections tolerate the swap + +**Answer: Yes — open connections survive; expect one duplicate `put`.** + +- **Connection survival:** downstream streams live in `envStreams`, keyed by `ScopedCredential`, + entirely independent of the upstream `clients` map. A re-anchor touches only `clients`, the rotator + anchor pointer, and the data store. An open client-side connection (keyed on env ID) keeps receiving + events across the swap (verified live: a `ping` still arrives after re-anchor). Connections are torn + down **only** for credentials that are actually removed (`removeCredential` → `RemoveCredential` → + `Close()`), which is the intended graceful-rotation behavior, not a swap side-effect. +- **Duplicate `put`:** the new anchor client's initial sync calls `store.Init(allData)`, which flows + through the store wrapper → `SendAllDataUpdate` → re-broadcast of a full `put`/`ping` to every + connected downstream stream. From a downstream SDK's perspective this is a duplicate put. It is + tolerable (SDKs apply puts idempotently) but **T2.c must expect it**; it is not corruption. + +### H3 — Big-segment sync after re-anchor + +**Answer: Re-wiring is required. It is NOT handled today.** + +`bigSegmentSync` is constructed once in `NewEnvContext`, wired to `envConfig.SDKKey` and `envConfig.EnvID` +at construction. The PoC confirms the swap path neither recreates the synchronizer nor informs it of the +new key (the `BigSegmentSynchronizer` interface has Start / HasSynced / SegmentUpdatesCh / Close — **no +credential-replacement method**). After a re-anchor it keeps polling/streaming big-segment data on the +**old** anchor key, which will break once the old key is revoked. + +**T2.d action:** add a re-wire path to `BigSegmentSynchronizer` (a `ReplaceCredential`-style method, +mirroring the event dispatcher / metrics publisher) **or** recreate the synchronizer on each re-anchor. +The "recreate" option is simpler; the "re-wire" option avoids dropping in-flight sync state. + +### H4 — `httpconfig` stays functional after re-anchor + +**Answer: Yes — no re-wire needed.** + +`httpconfig` carries TLS / proxy / transport / user-agent configuration plus the SDK key, but the only +key-dependent artifact is the `Authorization` default header on the pre-built `SDKHTTPConfig`. Relay +injects the *builder* (`SDKHTTPConfigFactory`), not the pre-built config, into `ld.Config.HTTP`, and the +SDK rebuilds the HTTP config with the new anchor key when it constructs the new client — so the +`Authorization` header is set correctly for the new anchor automatically. The pre-built `SDKHTTPConfig` / +`Client()` (used for event + big-segment transport) is key-independent except for that header, and those +components set their own auth per request rather than reading it from `httpconfig`. No action required. + +### H5 — Order of operations (start-new → swap pointer → close-old) + +**Answer: The recommended order is necessary but NOT sufficient for the in-memory store.** + +Because building the new client is what rebuilds (and empties) the in-memory store (H1), there is a +window after the swap in which evaluations see an empty store until the new anchor finishes its initial +sync — *regardless* of operation order. The PoC shows the env's store is replaced with a fresh, +uninitialized store as soon as the new client is registered. + +**Recommended remedy: hand the existing store over to the new client.** Because relay owns the store +implementation (it hands the SDK a single `storeAdapter`), the re-anchor can reuse the existing store for +the new client instead of letting `Build()` construct a fresh one — concretely, make +`SSERelayDataStoreAdapter.Build()` return its existing store when one is already present (or otherwise +seed the new client with the old client's store). The new anchor then reads populated, initialized data +immediately, so there is no empty-store window. Validated by +`TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow`. + +This is simpler than the alternatives originally considered — gating the swap on `Initialized()`, +mandating a persistent store, or otherwise decoupling store from client — and supersedes them. + +**Store-lifecycle caveat:** `streamUpdatesStoreWrapper.Close()` closes the underlying store. With +handover the retiring and new clients share one underlying store, so closing the retiring client must +**not** close it — the adapter (not the client) must own the store's lifecycle. This is not reproducible +with the fake client used in the PoC; verify against the real client in T2.c. + +**Recommended order, refined:** start new client (handing over the existing store) → swap anchor pointer +and re-wire peripherals → close old client, ensuring that close does not tear down the shared store. + +### H6 — Behavior during the swap window (requests arriving mid-swap) + +**Answer: Today there is a gap — `GetClient()` returns nil mid-swap.** + +`GetClient()` returns `clients[keyRotator.SDKKey()]`. In the current path the rotator's primary key flips +to the new key **synchronously** inside `UpdateCredential`, but the new client is created on a background +goroutine (`go startSDKClient`) and registered only afterward. The PoC deterministically observes +`GetClient() == nil` in that window (gated client factory, no sleeps). A request arriving mid-swap gets a +nil client. + +**T2.c action:** do not advance the anchor pointer until the new client is registered (and ideally +`Initialized()`). Combined with H5/H7, the rule is: **construct + initialize the new client first, then +atomically flip the anchor pointer.** + +### H7 — Failure mode: new client init fails + +**Answer: Today a failed re-anchor breaks the environment. Atomicity/rollback is required.** + +When the new anchor client fails to initialize, the rotator has already flipped the anchor pointer to the +new key, but no client exists for it — so `GetClient()` returns nil **even though the old anchor's client +is still alive and valid** during its grace period. The PoC confirms `GetInitError()` is set, `GetClient()` +is nil, and the old client is still present in the `clients` map (so the data path *could* have been +preserved). + +**T2.c action (this is §8's atomicity requirement):** validate that the new client initializes **before** +swapping the anchor pointer; on failure, roll back to the old anchor and preserve the previous accepted +set. Log a structured error and alarm (per §9). + +--- + +## Consolidated requirements for T2.c / T2.d + +| # | Requirement | From | +|---|---|---| +| 1 | Construct + initialize the new anchor client **before** flipping the anchor pointer; flip atomically. | H5, H6 | +| 2 | On new-client init failure, roll back to the old anchor; preserve previous accepted set; log + alarm. | H7 | +| 3 | Hand the existing store over to the new client (make `SSERelayDataStoreAdapter.Build` reuse its store) so there is no empty-store window; ensure the retiring client's `Close()` does not tear down the shared store. | H1, H5 | +| 4 | Re-wire big-segment sync on re-anchor (add a replace-credential method, or recreate the synchronizer). | H3 | +| 5 | Continue calling `ReplaceCredential` on the event dispatcher + metrics publisher (already wired in `addCredential`). | §7 table | +| 6 | Expect a duplicate downstream `put` from the new anchor's initial sync; ensure downstream connections are not torn down for retained credentials. | H2 | +| 7 | No `httpconfig` change needed. | H4 | + +## What did NOT need changing + +- `httpconfig` (H4). +- Downstream SSE routing / `envStreams` (H2) — already credential-scoped and independent of the anchor. +- Event dispatcher + metrics publisher already expose `ReplaceCredential` and are already called from + `addCredential` on an SDK-key change. + +## Test inventory + +All tests are in [`internal/relayenv/env_context_reanchor_test.go`](../../internal/relayenv/env_context_reanchor_test.go), +prefixed `TestReanchorPoC_H_…`: + +- `H1_SharedStoreAdapterRebuildSemantics` — in-memory rebuild vs. persistent-store preservation. +- `H2_DownstreamConnectionSurvivesReAnchor` — live client-side connection survives the swap. +- `H2_NewClientInitialSyncRebroadcastsPut` — duplicate `put` is produced and counted. +- `H3_BigSegmentSyncIsNotReWiredOnReAnchor` — synchronizer keeps the old key; no re-wire today. +- `H4_HTTPConfigIsKeyIndependentExceptAuthHeader` — only the auth header is key-dependent. +- `H5_InMemoryStoreIsWipedByReAnchor` — store replaced/empty after swap. +- `H5_StoreHandoverAvoidsEmptyWindow` — reusing the store across the swap avoids the empty window (the remedy). +- `H6_AnchorPointerFlipsBeforeNewClientIsRegistered` — `GetClient()` nil mid-swap (deterministic). +- `H7_FailedNewClientLeavesEnvWithoutAnchorClient` — failed swap breaks the env; old client still alive. diff --git a/docs/concurrent-keys/phase1-design.md b/.agent-docs/concurrent-keys/phase1-design.md similarity index 92% rename from docs/concurrent-keys/phase1-design.md rename to .agent-docs/concurrent-keys/phase1-design.md index 02b12e23..df01c026 100644 --- a/docs/concurrent-keys/phase1-design.md +++ b/.agent-docs/concurrent-keys/phase1-design.md @@ -243,33 +243,31 @@ This is the highest-risk piece of Phase 1. The **T0 PoC** validated the swap mec ### Required order of operations ``` -1. Build the new anchor's SDK client (do not flip the anchor pointer yet). +1. Build the new anchor's SDK client, handing over the existing data store (do not flip the anchor pointer yet). 2. Wait for the new client to report Initialized() == true. 3. Atomically flip the rotator's anchor pointer. 4. Call ReplaceCredential on event dispatcher + metrics publisher. 5. Re-wire (or recreate) big-segment sync. -6. Close the old anchor's client (after its grace period elapses for downstream traffic). +6. Close the old anchor's client (after its grace period elapses for downstream traffic), ensuring its Close() does not tear down the now-shared store. ``` -This order is *necessary* — the PoC found that flipping the pointer too early leaves `GetClient()` nil mid-swap (H6) and breaks the env on init failure (H7) — and *sufficient* only with the store-handling caveat below. +This order is *necessary* — the PoC found that flipping the pointer too early leaves `GetClient()` nil mid-swap (H6) and breaks the env on init failure (H7) — and *sufficient* only with the store-handling approach below. -### The data store: old stays authoritative until new is `Initialized()` +### The data store: hand the existing store over to the new client -An earlier version of this design assumed two SDK clients pointed at the same env would feed the *same* data store as a side-effect. The PoC (H1, H5) showed this is **wrong for the in-memory store**: each SDK client construction calls `storeAdapter.Build()`, which atomically swaps in a *new, empty* store. The new client must re-sync from scratch. +An earlier version of this design assumed two SDK clients pointed at the same env would feed the *same* data store as a side-effect. The PoC (H1, H5) showed this is **wrong for the in-memory store**: each SDK client construction calls `storeAdapter.Build()`, which atomically swaps in a *new, empty* store, so the new client would otherwise have to re-sync from scratch (an empty-store window). This affects only the in-memory case; with a persistent store (Redis, DynamoDB) the data lives outside the wrapper and survives the swap. -This affects only the in-memory case. With a persistent store (Redis, DynamoDB), the data lives outside the wrapper and survives the swap. +**Chosen remedy: hand the existing store over to the new client.** Because relay owns the store implementation (it hands the SDK a single `storeAdapter`), the re-anchor reuses the existing store for the new client instead of letting `Build()` construct a fresh one — concretely, make `SSERelayDataStoreAdapter.Build()` return its existing store when one is already present (or otherwise seed the new client with the old client's store). The new anchor then serves populated, initialized data immediately, with no empty-store window. Validated by `TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow`. -Three remedies were considered (PoC findings §H5): -1. **Keep the old store/anchor authoritative until the new client is `Initialized()`.** ← **chosen** -2. Require a persistent store for graceful re-anchor. -3. Decouple the data store lifecycle from the client lifecycle. +This is the concrete form of decoupling the store's lifecycle from the client's. Two alternatives were considered and rejected as heavier: gating the swap on the new client reaching `Initialized()` (still leaves a window for already-connected reads and keeps the store coupled to client construction), and mandating a persistent store for graceful re-anchor (constrains deployments). -Option 1 aligns with §8's "superset during transition" principle: just as the *accepted credential set* is a superset during a keys-change event, the *store* (and the client serving it) stays in place until the replacement is ready. +**Store-lifecycle caveat:** `streamUpdatesStoreWrapper.Close()` closes the underlying store. With handover the retiring and new clients share one underlying store, so closing the retiring client must **not** close it — the adapter (not the client) must own the store's lifecycle. (Not reproducible with the fake client used in the PoC; verify against the real client in T2.c.) ### Component re-wiring on re-anchor | Component | Today | On re-anchor | |---|---|---| +| Data store | Rebuilt per client by `storeAdapter.Build()` | Hand the existing store over (adapter reuses it); the retiring client's `Close()` must not tear it down (T2.c) | | Event dispatcher | Has `ReplaceCredential` | Call `ReplaceCredential` (T2.c) | | Metrics publisher | Has `ReplaceCredential` | Call `ReplaceCredential` (T2.c) | | Big-segment sync | Wired at construction; no replacement method | Re-wire (add a replace-credential method) **or** recreate the synchronizer (T2.d). Recreate is simpler; re-wire preserves in-flight sync state. | @@ -286,7 +284,7 @@ If the new client fails to initialize, the swap **rolls back**: the rotator's an |---|---|---|---| | 1 | Build + initialize the new anchor client *before* flipping the pointer; flip atomically. | H5, H6 | T2.c | | 2 | On init failure, roll back to old anchor; preserve previous accepted set; log + alarm. | H7 | T2.c | -| 3 | Keep the old store/anchor authoritative until new client `Initialized()`. | H1, H5 | T2.c | +| 3 | Hand the existing store over to the new client (adapter reuses its store); ensure the retiring client's `Close()` does not tear down the shared store. | H1, H5 | T2.c | | 4 | Re-wire big-segment sync on re-anchor (recreate or replace-credential). | H3 | T2.d | | 5 | Call `ReplaceCredential` on event dispatcher + metrics publisher. | §7 | T2.c (already wired in `addCredential`) | | 6 | Expect duplicate downstream `put`; retain connections for credentials still in the accepted set. | H2 | T2.c (awareness) | diff --git a/docs/concurrent-keys/phase1-plan.md b/.agent-docs/concurrent-keys/phase1-plan.md similarity index 97% rename from docs/concurrent-keys/phase1-plan.md rename to .agent-docs/concurrent-keys/phase1-plan.md index 4b52aec2..27ca8bfa 100644 --- a/docs/concurrent-keys/phase1-plan.md +++ b/.agent-docs/concurrent-keys/phase1-plan.md @@ -22,7 +22,7 @@ This document covers *how* we ship Phase 1: branching, sequencing, tasks, depend git worktree add ../ld-relay-wt-feat-concurrent-keys -b feat/concurrent-keys v8 ``` -This document is committed in that worktree at `docs/concurrent-keys/phase1-plan.md`. +This document is committed in that worktree at `.agent-docs/concurrent-keys/phase1-plan.md`. --- @@ -78,7 +78,7 @@ Each task has: ticket name, files touched, dependencies, estimates. Acceptance c | Task | Files | Depends on | Human | AI agent | |---|---|---|---|---| -| **T5.f** — Code cleanup: remove project scaffolding before v8 merge | `docs/concurrent-keys/` (entire directory), `internal/relayenv/env_context_reanchor_test.go`, anything else project-specific | All Wave 2 terminal sub-tasks (T1.c, T2.b, T2.d, T2.e, T4) | 0.5-1 day | 30 min - 1 hr | +| **T5.f** — Code cleanup: remove project scaffolding before v8 merge | `.agent-docs/concurrent-keys/` (entire directory), `internal/relayenv/env_context_reanchor_test.go`, anything else project-specific | All Wave 2 terminal sub-tasks (T1.c, T2.b, T2.d, T2.e, T4) | 0.5-1 day | 30 min - 1 hr | | **T5.g** — Merge `feat/concurrent-keys` to v8 + publish release | None (release activity) | T5.f | 0.5-1 day | n/a (release task, not coding) | | **T5.e** — Merge-forward to v9 | `internal/relayenv/*`, streaming path | T5.g (and calendar — may be weeks/months after the v8 release) | 3-7 days | 1-2 days (with iteration) | @@ -143,7 +143,7 @@ Today: `rotator.go:168-169` panics with `"programmer error: mobile keys do not s Internal fields only. No API change. Existing public methods (`SDKKey()`, `GetCredentials()`, etc.) continue to return what they return today by reading from the new internal state where the single primary maps to a one-element set. -Reviewer-friendly comment to add at the top of the new fields: `// Consumed by T1.b (ReconcileCredentials API). See docs/concurrent-keys/phase1-design.md §6.2.` +Reviewer-friendly comment to add at the top of the new fields: `// Consumed by T1.b (ReconcileCredentials API). See .agent-docs/concurrent-keys/phase1-design.md §6.2.` ### T1.b — `ReconcileCredentials` API @@ -281,7 +281,7 @@ Capture upstream payloads from v8 under realistic SDK traffic. Assert post-Phase Remove all project-specific scaffolding from `feat/concurrent-keys` *before* T5.g merges the branch to v8. The canonical design + plan docs and the PoC test file were useful during development; they shouldn't land on v8. What to remove: -- `docs/concurrent-keys/` — entire directory (this file is one of the things being removed). Save off-branch if you want to keep it for reference. +- `.agent-docs/concurrent-keys/` — entire directory (this file is one of the things being removed). Save off-branch if you want to keep it for reference. - `internal/relayenv/env_context_reanchor_test.go` — PoC test file. Verify any useful tests have already been adopted into proper regression test files by T2.c before deleting. - Any other concurrent-keys-specific scaffolding that may have accumulated. @@ -452,7 +452,7 @@ SDK-2453 (Epic) — Relay Proxy Multi Keys Support |---|---| | Feature branch? | `feat/concurrent-keys` off v8 | | Sub-PR branches? | `aaronz//` off the feature branch (use the specific sub-task ticket ID, not the epic SDK-2453) | -| Where do canonical docs live? | This file + `phase1-design.md` in `docs/concurrent-keys/` on the feature branch | +| Where do canonical docs live? | This file + `phase1-design.md` in `.agent-docs/concurrent-keys/` on the feature branch | | Where do working notes live? | `docs/agents/phase1-*.md` in the design worktree (gitignored, not on this branch) | | How is ordering enforced within a `keys change` event? | Add → re-anchor → remove (atomic) | | What triggers re-anchor? | `sdkKey.value` changed | diff --git a/internal/relayenv/env_context_reanchor_test.go b/internal/relayenv/env_context_reanchor_test.go new file mode 100644 index 00000000..e5436147 --- /dev/null +++ b/internal/relayenv/env_context_reanchor_test.go @@ -0,0 +1,622 @@ +package relayenv + +// T0 — Re-anchoring PoC (SDK-2453 / SDK-2530). +// +// These tests validate the upstream SDK-client swap mechanism that T2.c will implement. Each test +// answers one of the seven hypotheses in .agent-docs/concurrent-keys/phase1-design.md §7. They are +// written as durable, executable probes of today's primitives so they survive into T2 as regression +// tests and as the executable spec for the re-anchor implementation. +// +// A written summary of the findings lives in +// .agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md. +// +// Terminology: "re-anchor" = swapping the single upstream SDK client when sdkKey.value changes. +// Today there is no dedicated re-anchor method; the closest existing path is UpdateCredential with a +// grace period (which rotates the primary SDK key and stands up a new client), so several tests drive +// that path and observe where it falls short of the §7 requirements. + +import ( + "errors" + "net/http" + "sync" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/basictypes" + "github.com/launchdarkly/ld-relay/v8/internal/bigsegments" + "github.com/launchdarkly/ld-relay/v8/internal/httpconfig" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" + st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + "github.com/launchdarkly/ld-relay/v8/internal/store" + "github.com/launchdarkly/ld-relay/v8/internal/streams" + + "github.com/launchdarkly/eventsource" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + ld "github.com/launchdarkly/go-server-sdk/v7" + "github.com/launchdarkly/go-server-sdk/v7/ldcomponents" + "github.com/launchdarkly/go-server-sdk/v7/subsystems" + "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" + "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoretypes" + helpers "github.com/launchdarkly/go-test-helpers/v3" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// reanchorTestKey2 is the "new anchor" SDK key we re-anchor onto. It is deliberately NOT in the real +// sdk- credential format so it won't trip secret scanners; relay treats SDK keys as opaque +// non-empty strings, so any value works here. +const reanchorTestKey2 = config.SDKKey("reanchor-poc-new-anchor") + +// recordingStreamUpdates is a streams.EnvStreamUpdates that counts the broadcasts it receives, so we +// can observe whether a re-anchor produces duplicate downstream "put"s. +type recordingStreamUpdates struct { + mu sync.Mutex + allDataUpdates int + singleUpdates int + invalidations int +} + +func (r *recordingStreamUpdates) SendAllDataUpdate(_ []ldstoretypes.Collection) { + r.mu.Lock() + r.allDataUpdates++ + r.mu.Unlock() +} + +func (r *recordingStreamUpdates) SendSingleItemUpdate(_ ldstoretypes.DataKind, _ string, _ ldstoretypes.ItemDescriptor) { + r.mu.Lock() + r.singleUpdates++ + r.mu.Unlock() +} + +func (r *recordingStreamUpdates) InvalidateClientSideState() { + r.mu.Lock() + r.invalidations++ + r.mu.Unlock() +} + +func (r *recordingStreamUpdates) allDataCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.allDataUpdates +} + +// sharedStoreFactory is a DataStore configurer that hands back the SAME underlying store instance on +// every Build call. It models a persistent store (Redis/DynamoDB), where the data lives outside the +// process and survives the recreation of the wrapping store on a client swap. +type sharedStoreFactory struct { + store subsystems.DataStore +} + +func (f *sharedStoreFactory) Build(_ subsystems.ClientContext) (subsystems.DataStore, error) { + return f.store, nil +} + +// newReanchorCredentialUpdate builds a CredentialUpdate that rotates the primary SDK key to newKey +// while keeping oldKey valid for a grace hour (so the old client is not torn down during the swap). +// This mirrors the backend's default-rotation behavior: the new anchor is non-expiring, the demoted +// old anchor carries an expiry. +func newReanchorCredentialUpdate(newKey, oldKey config.SDKKey, now time.Time) *CredentialUpdate { + return NewCredentialUpdate(newKey).WithTime(now).WithGracePeriod(oldKey, now.Add(time.Hour)) +} + +// ----------------------------------------------------------------------------------------------- +// Hypothesis 1: Two SDK clients sharing a storeAdapter don't corrupt store invariants. +// ----------------------------------------------------------------------------------------------- + +func TestReanchorPoC_H1_SharedStoreAdapterRebuildSemantics(t *testing.T) { + featureKind := ldstoreimpl.Features() + flagKey := st.Flag1ServerSide.Flag.Key + + // The design (§7) assumes "two SDK clients pointed at the same env can feed the same store as a + // side-effect." This sub-test shows that assumption is FALSE for the default in-memory store: each + // client init calls storeAdapter.Build, which constructs a brand-new wrapper around a brand-new + // underlying store and atomically swaps it in. No corruption occurs, but the new client starts from + // an empty store. + t.Run("in-memory factory builds a fresh empty store on each client init", func(t *testing.T) { + rec := &recordingStreamUpdates{} + adapter := store.NewSSERelayDataStoreAdapter(ldcomponents.InMemoryDataStore(), rec) + + // First client init populates the store, as the original anchor's client would. + s1, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + require.NoError(t, s1.Init(st.AllData)) + require.Same(t, s1, adapter.GetStore()) + + got, err := adapter.GetStore().Get(featureKind, flagKey) + require.NoError(t, err) + require.NotNil(t, got.Item, "data should be present after the first client's sync") + + // Second client init = the re-anchor's "start new client" step. + s2, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + + // FINDING: Build swaps in a new store instance... + assert.NotSame(t, s1, s2, "each Build creates a new store wrapper") + assert.Same(t, s2, adapter.GetStore(), "the adapter now points at the new store") + + // ...and that store is empty + uninitialized. So the two clients do NOT share data through the + // in-memory store; the new anchor must re-sync from scratch. + assert.False(t, adapter.GetStore().IsInitialized(), "the new in-memory store starts uninitialized") + got2, err := adapter.GetStore().Get(featureKind, flagKey) + require.NoError(t, err) + assert.Nil(t, got2.Item, "the new in-memory store starts empty") + }) + + // With a persistent store, the underlying data lives outside the wrapper, so the swap preserves it. + // This is the configuration in which the §7 "shared store" assumption actually holds. + t.Run("shared (persistent) underlying store preserves data across client init", func(t *testing.T) { + underlying, err := ldcomponents.InMemoryDataStore().Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + rec := &recordingStreamUpdates{} + adapter := store.NewSSERelayDataStoreAdapter(&sharedStoreFactory{store: underlying}, rec) + + s1, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + require.NoError(t, s1.Init(st.AllData)) + + // Re-anchor's "start new client" step. + s2, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + + // The wrapper is new, but it wraps the SAME underlying store, so data + initialization survive. + assert.True(t, s2.IsInitialized(), "persistent store stays initialized across the swap") + got, err := s2.Get(featureKind, flagKey) + require.NoError(t, err) + assert.NotNil(t, got.Item, "data survives the swap when the underlying store is shared") + }) +} + +// ----------------------------------------------------------------------------------------------- +// Hypothesis 2: Downstream SSE connections tolerate the swap. +// 2a: an open downstream connection survives a re-anchor and keeps receiving events. +// 2b: the new anchor's initial sync re-broadcasts a (duplicate) "put" downstream. +// ----------------------------------------------------------------------------------------------- + +func TestReanchorPoC_H2_DownstreamConnectionSurvivesReAnchor(t *testing.T) { + envConfig := st.EnvClientSide.Config + + fakeBigSegmentStoreFactory := func(config.EnvConfig, config.Config, ldlog.Loggers) (bigsegments.BigSegmentStore, error) { + return bigsegments.NewNullBigSegmentStore(), nil + } + fakeSynchronizerFactory := &mockBigSegmentSynchronizerFactory{} + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + jsClientStreams := streams.NewStreamProvider(basictypes.JSClientPingStream, time.Hour, 0) + clientCh := make(chan *testclient.FakeLDClient, 10) + sdkStartedCh := make(chan EnvContext, 10) + env, err := NewEnvContext(EnvContextImplParams{ + Identifiers: EnvIdentifiers{ConfiguredName: st.EnvClientSide.Name}, + EnvConfig: envConfig, + AllConfig: config.Config{}, + BigSegmentStoreFactory: fakeBigSegmentStoreFactory, + BigSegmentSynchronizerFactory: fakeSynchronizerFactory.create, + ClientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientCh), + SDKBigSegmentsConfigFactory: ldcomponents.BigSegments( + st.ExistingInstance[subsystems.BigSegmentStore](&st.NoOpSDKBigSegmentStore{}), + ), + StreamProviders: []streams.StreamProvider{jsClientStreams}, + ConnectionMapper: mockConnectionMapper{}, + Loggers: mockLog.Loggers, + }, sdkStartedCh) + require.NoError(t, err) + defer env.Close() + + synchronizer := fakeSynchronizerFactory.synchronizer + require.NotNil(t, synchronizer) + + // Wait for the original anchor client and initialize the store so the client-side stream is ready. + <-sdkStartedCh + require.NoError(t, env.GetStore().Init(nil)) + + streamHandler := env.GetStreamHandler(jsClientStreams, envConfig.EnvID) + req, _ := http.NewRequest("GET", "", nil) + st.WithStreamRequest(t, req, streamHandler, func(eventCh <-chan eventsource.Event) { + initEvent := helpers.RequireValue(t, eventCh, time.Minute) + assert.Equal(t, "ping", initEvent.Event()) + if !helpers.AssertNoMoreValues(t, eventCh, 100*time.Millisecond) { + t.FailNow() + } + + // --- Re-anchor while the downstream connection is open. --- + // The connection is keyed on the env ID (a ScopedCredential), independent of the upstream SDK + // key, so swapping the SDK anchor must not disturb it. + start := time.Unix(1000, 0) + env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + + // The new anchor client comes up on a background goroutine. Credential additions start the client + // with a nil readyCh (so it does NOT signal sdkStartedCh); wait on the credential set instead. + require.Eventually(t, func() bool { + creds := env.GetCredentials() + for _, c := range creds { + if c == reanchorTestKey2 { + return true + } + } + return false + }, time.Second, 10*time.Millisecond, "rotation to the new anchor should be applied") + + // FINDING: the open client-side connection survives the swap and still delivers events. + synchronizer.updateCh <- bigsegments.UpdatesSummary{SegmentKeysUpdated: []string{"fake-segment-key"}} + pingEvent := helpers.RequireValue(t, eventCh, time.Second) + assert.Equal(t, "ping", pingEvent.Event(), "downstream connection should survive the re-anchor") + }) +} + +func TestReanchorPoC_H2_NewClientInitialSyncRebroadcastsPut(t *testing.T) { + rec := &recordingStreamUpdates{} + adapter := store.NewSSERelayDataStoreAdapter(ldcomponents.InMemoryDataStore(), rec) + + // Original anchor client builds and performs its initial sync -> one downstream "put". + s1, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + require.NoError(t, s1.Init(st.AllData)) + require.Equal(t, 1, rec.allDataCount()) + + // Re-anchor: the new anchor client builds a fresh store and performs its OWN initial sync. + s2, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + require.NoError(t, s2.Init(st.AllData)) + + // FINDING: the new anchor's initial sync produces a second full "put" to every connected downstream + // stream. From a downstream SDK's perspective this is a duplicate put. It is tolerable (SDKs apply + // puts idempotently) but T2.c must expect it; it is not a corruption. + assert.Equal(t, 2, rec.allDataCount(), "the new anchor's initial sync re-broadcasts a full put") +} + +// ----------------------------------------------------------------------------------------------- +// Hypothesis 3: Big-segment sync after re-anchor. +// ----------------------------------------------------------------------------------------------- + +// capturingBigSegmentSynchronizerFactory records the SDK key it was constructed with and how many +// times it was invoked, so we can detect whether a re-anchor re-wires big-segment sync. +type capturingBigSegmentSynchronizerFactory struct { + mu sync.Mutex + createCount int + lastSDKKey config.SDKKey + synchronizer *mockBigSegmentSynchronizer +} + +func (f *capturingBigSegmentSynchronizerFactory) create( + _ httpconfig.HTTPConfig, + _ bigsegments.BigSegmentStore, + _ string, + _ string, + _ config.EnvironmentID, + sdkKey config.SDKKey, + _ ldlog.Loggers, + _ string, +) bigsegments.BigSegmentSynchronizer { + f.mu.Lock() + defer f.mu.Unlock() + f.createCount++ + f.lastSDKKey = sdkKey + f.synchronizer = &mockBigSegmentSynchronizer{updateCh: make(chan bigsegments.UpdatesSummary)} + return f.synchronizer +} + +func (f *capturingBigSegmentSynchronizerFactory) snapshot() (int, config.SDKKey) { + f.mu.Lock() + defer f.mu.Unlock() + return f.createCount, f.lastSDKKey +} + +func TestReanchorPoC_H3_BigSegmentSyncIsNotReWiredOnReAnchor(t *testing.T) { + envConfig := st.EnvMain.Config + + fakeBigSegmentStoreFactory := func(config.EnvConfig, config.Config, ldlog.Loggers) (bigsegments.BigSegmentStore, error) { + return bigsegments.NewNullBigSegmentStore(), nil + } + capturing := &capturingBigSegmentSynchronizerFactory{} + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + env, err := NewEnvContext(EnvContextImplParams{ + Identifiers: EnvIdentifiers{ConfiguredName: st.EnvMain.Name}, + EnvConfig: envConfig, + AllConfig: config.Config{}, + BigSegmentStoreFactory: fakeBigSegmentStoreFactory, + BigSegmentSynchronizerFactory: capturing.create, + ClientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientCh), + SDKBigSegmentsConfigFactory: ldcomponents.BigSegments( + st.ExistingInstance[subsystems.BigSegmentStore](&st.NoOpSDKBigSegmentStore{}), + ), + ConnectionMapper: mockConnectionMapper{}, + Loggers: mockLog.Loggers, + }, nil) + require.NoError(t, err) + defer env.Close() + + count, sdkKey := capturing.snapshot() + require.Equal(t, 1, count, "the synchronizer is constructed once at env creation") + require.Equal(t, envConfig.SDKKey, sdkKey, "it is wired to the original anchor's SDK key") + + // Re-anchor onto a new SDK key. + start := time.Unix(1000, 0) + env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + require.Eventually(t, func() bool { + for _, c := range env.GetCredentials() { + if c == reanchorTestKey2 { + return true + } + } + return false + }, time.Second, 10*time.Millisecond, "rotation to the new anchor should be applied") + + // Give any (hypothetical) re-wire a chance to run. + require.Never(t, func() bool { + c, _ := capturing.snapshot() + return c != 1 + }, 200*time.Millisecond, 20*time.Millisecond, "synchronizer must not be recreated by the re-anchor") + + // FINDING: big-segment sync is wired to the SDK key at construction and is NOT re-wired by today's + // swap path -- the synchronizer is neither recreated nor told about the new key (the + // BigSegmentSynchronizer interface has no credential-replacement method). After re-anchor it keeps + // polling/streaming on the OLD anchor key. T2.d must add a re-wire path (a ReplaceCredential-style + // method) or recreate the synchronizer on each re-anchor. + count, sdkKey = capturing.snapshot() + assert.Equal(t, 1, count, "synchronizer was not recreated on re-anchor") + assert.Equal(t, envConfig.SDKKey, sdkKey, "synchronizer still references the old anchor key") +} + +// ----------------------------------------------------------------------------------------------- +// Hypothesis 4: httpconfig stays functional after re-anchor. +// ----------------------------------------------------------------------------------------------- + +func TestReanchorPoC_H4_HTTPConfigIsKeyIndependentExceptAuthHeader(t *testing.T) { + loggers := ldlog.NewDisabledLoggers() + key1 := config.SDKKey("sdk-key-one") + key2 := config.SDKKey("sdk-key-two") + + var proxy config.ProxyConfig + var httpC config.HTTPConfig + + c1, err := httpconfig.NewHTTPConfig(proxy, httpC, key1, "user-agent", loggers) + require.NoError(t, err) + c2, err := httpconfig.NewHTTPConfig(proxy, httpC, key2, "user-agent", loggers) + require.NoError(t, err) + + // The ONLY key-dependent artifact is the Authorization default header on the pre-built SDK HTTP + // config. + assert.Equal(t, string(key1), c1.SDKHTTPConfig.DefaultHeaders.Get("Authorization")) + assert.Equal(t, string(key2), c2.SDKHTTPConfig.DefaultHeaders.Get("Authorization")) + + // Everything else (proxy settings, user agent, and the rest of the default headers) is identical + // and key-independent. + h1 := c1.SDKHTTPConfig.DefaultHeaders.Clone() + h2 := c2.SDKHTTPConfig.DefaultHeaders.Clone() + h1.Del("Authorization") + h2.Del("Authorization") + assert.Equal(t, h1, h2, "non-auth default headers are key-independent") + assert.Equal(t, c1.ProxyConfig, c2.ProxyConfig, "proxy config is key-independent") + + // FINDING: httpconfig needs NO re-wire on re-anchor. Relay injects the *builder* + // (SDKHTTPConfigFactory) into ld.Config.HTTP, and the SDK rebuilds the HTTP config with the new + // anchor key when it constructs the new client, so the Authorization header is set correctly for the + // new anchor automatically. The pre-built SDKHTTPConfig / Client() (used for event + big-segment + // transport) is key-independent except for that Authorization header, which those components set per + // request from their own credential rather than reading it from httpconfig. +} + +// ----------------------------------------------------------------------------------------------- +// Hypothesis 5: Order of operations / the in-memory store window. +// ----------------------------------------------------------------------------------------------- + +func TestReanchorPoC_H5_InMemoryStoreIsWipedByReAnchor(t *testing.T) { + featureKind := ldstoreimpl.Features() + flagKey := st.Flag1ServerSide.Flag.Key + envConfig := st.EnvMain.Config + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactoryWithChannel(true, clientCh), mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + client1 := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) + + // Populate the store as the original anchor's client would have via its stream sync. + require.NoError(t, env.GetStore().Init(st.AllData)) + oldStore := env.GetStore() + got, err := oldStore.Get(featureKind, flagKey) + require.NoError(t, err) + require.NotNil(t, got.Item) + + // Re-anchor onto a new key (old key kept valid for a grace hour, so the old client is not closed -- + // i.e. this exercises the recommended "start-new-before-close-old" ordering). + start := time.Unix(1000, 0) + env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + + client2 := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond, + "GetClient should return the new anchor's client once it is registered") + + // FINDING: starting the new client replaced the data store with a fresh, empty, uninitialized one. + // This happens regardless of operation order, because building the new client is what rebuilds the + // store. So "start-new -> swap-pointer -> close-old" alone is NOT sufficient with an in-memory store: + // there is a window in which evaluations see an empty store until the new anchor finishes its initial + // sync. T2.c must either (a) keep the old store/anchor authoritative until the new client reports + // Initialized()==true, (b) require a persistent store for graceful re-anchor, or (c) decouple the + // data store lifecycle from the client lifecycle so a new client does not rebuild it. + newStore := env.GetStore() + assert.NotSame(t, oldStore, newStore, "the data store instance was replaced by the new client") + assert.False(t, newStore.IsInitialized(), "the new store is uninitialized until the new anchor re-syncs") + got2, err := newStore.Get(featureKind, flagKey) + require.NoError(t, err) + assert.Nil(t, got2.Item, "data is absent in the new store until the new anchor re-syncs") +} + +// TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow validates the reviewer suggestion that, because +// relay owns the data store implementation (it hands the SDK a single storeAdapter), the re-anchor can +// hand the existing store over to the new client instead of letting it build a fresh one. Modeled here +// by a DataStoreFactory that returns the same underlying store on every Build; the production change +// (T2.c/T2.d) is to make SSERelayDataStoreAdapter reuse its store across the swap. With handover the +// new anchor's client sees the populated, initialized store immediately -- no empty-store window +// (contrast TestReanchorPoC_H5_InMemoryStoreIsWipedByReAnchor). +// +// CAVEAT for the implementation (not reproducible with the fake client, so documented here and in the +// findings): streamUpdatesStoreWrapper.Close() closes the underlying store. If the new client wraps the +// SAME underlying store, closing the retiring client must NOT close it -- the store's lifecycle has to +// be owned by the adapter, not by the client being retired. +func TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow(t *testing.T) { + featureKind := ldstoreimpl.Features() + flagKey := st.Flag1ServerSide.Flag.Key + envConfig := st.EnvMain.Config + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + // A store factory that hands the same underlying store to every client (the "handover" model). + underlying, err := ldcomponents.InMemoryDataStore().Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + handoverFactory := &sharedStoreFactory{store: underlying} + + clientCh := make(chan *testclient.FakeLDClient, 10) + readyCh := make(chan EnvContext, 1) + env, err := NewEnvContext(EnvContextImplParams{ + Identifiers: EnvIdentifiers{ConfiguredName: envName}, + EnvConfig: envConfig, + ClientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientCh), + DataStoreFactory: handoverFactory, + ConnectionMapper: mockConnectionMapper{}, + Loggers: mockLog.Loggers, + }, readyCh) + require.NoError(t, err) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + client1 := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) + + // Populate the store as the original anchor's client would have. + require.NoError(t, env.GetStore().Init(st.AllData)) + + // Re-anchor onto a new key. + start := time.Unix(1000, 0) + env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + + client2 := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond) + + // FINDING: with store handover there is no empty-store window -- the new client's store is still + // initialized and still holds the data, because the underlying store was reused rather than rebuilt. + newStore := env.GetStore() + assert.True(t, newStore.IsInitialized(), "handed-over store stays initialized across the re-anchor") + got, err := newStore.Get(featureKind, flagKey) + require.NoError(t, err) + assert.NotNil(t, got.Item, "data is preserved across the re-anchor when the store is handed over") +} + +// ----------------------------------------------------------------------------------------------- +// Hypothesis 6: Behavior during the swap window (requests arriving mid-swap). +// ----------------------------------------------------------------------------------------------- + +func TestReanchorPoC_H6_AnchorPointerFlipsBeforeNewClientIsRegistered(t *testing.T) { + envConfig := st.EnvMain.Config + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + inner := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + // gate blocks construction of the NEW anchor's client so we can observe the swap window + // deterministically (no sleeps / no racing). + gate := make(chan struct{}) + gatedFactory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == reanchorTestKey2 { + <-gate + } + return inner(sdkKey, cfg, timeout) + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, gatedFactory, mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + client1 := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) + + // Re-anchor. UpdateCredential flips the rotator's primary SDK key synchronously, then starts the new + // client on a background goroutine (which blocks in the factory on `gate`). + start := time.Unix(1000, 0) + env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + + // FINDING: there is a window where the anchor pointer already names the new key but no client exists + // for it yet, so GetClient() returns nil. GetClient() == clients[rotator.SDKKey()], and the rotator's + // primary flipped to the new key before startSDKClient registered the client. A request arriving in + // this window gets a nil client. T2.c must not advance the anchor pointer until the new client is + // registered (and ideally Initialized()). + assert.Nil(t, env.GetClient(), "GetClient() is nil during the swap window") + + // Release the gate; the new client registers and GetClient() recovers. + close(gate) + client2 := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond, + "GetClient() recovers once the new client is registered") +} + +// ----------------------------------------------------------------------------------------------- +// Hypothesis 7: Failure modes — new client init fails. +// ----------------------------------------------------------------------------------------------- + +func TestReanchorPoC_H7_FailedNewClientLeavesEnvWithoutAnchorClient(t *testing.T) { + envConfig := st.EnvMain.Config + fakeErr := errors.New("new anchor client failed to initialize") + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + inner := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + // Succeed for the original anchor; fail for the new anchor. + failingFactory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == reanchorTestKey2 { + return nil, fakeErr + } + return inner(sdkKey, cfg, timeout) + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, failingFactory, mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + client1 := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) + + // Re-anchor onto a key whose client init fails (old key kept valid for a grace hour). + start := time.Unix(1000, 0) + env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + + require.Eventually(t, func() bool { return env.GetInitError() != nil }, time.Second, 10*time.Millisecond, + "the failed new-client init should surface as an init error") + assert.Equal(t, fakeErr, env.GetInitError()) + + // FINDING: the rotator already flipped the anchor to the new key, but no client exists for it, so + // GetClient() returns nil -- even though the OLD anchor's client is still alive and valid during its + // grace period. A failed re-anchor breaks the environment with today's code. This is exactly the §8 + // atomicity requirement: T2.c must validate that the new client initializes BEFORE swapping the + // anchor pointer, and roll back to the old anchor on failure (preserving the previous accepted set). + assert.Nil(t, env.GetClient(), "GetClient() is nil after a failed re-anchor") + + envImpl := env.(*envContextImpl) + envImpl.mu.RLock() + _, oldStillPresent := envImpl.clients[envConfig.SDKKey] + envImpl.mu.RUnlock() + assert.True(t, oldStillPresent, + "the old anchor's client is still alive -- the data path could have been preserved by rolling back") +} From 32724ee39b98b2dc53a8bae2096b80c8d3a343ec Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Sun, 21 Jun 2026 21:31:15 -0700 Subject: [PATCH 12/66] docs(concurrent-keys): add end-to-end acceptance scenario catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds §7 "End-to-end acceptance scenarios" — 22 scenarios across multi-key auth, re-anchoring, key lifecycle, defensive behavior, sources, back-compat, and observability. Each scenario names the owning sub-task; the catalog as a whole is the release-readiness coverage check (referenced from the release-readiness checklist). Will be reflected in the project's HTML progress dashboard in a separate session. --- .agent-docs/concurrent-keys/phase1-plan.md | 69 +++++++++++++++++++++- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/.agent-docs/concurrent-keys/phase1-plan.md b/.agent-docs/concurrent-keys/phase1-plan.md index 27ca8bfa..a47d4a90 100644 --- a/.agent-docs/concurrent-keys/phase1-plan.md +++ b/.agent-docs/concurrent-keys/phase1-plan.md @@ -358,16 +358,79 @@ These live in T5 and run continuously: - **T5.a (test harness)**: enables the per-sub-task tests above. - **T5.b (events payload regression)**: catches schema drift in event payloads. +### End-to-end acceptance scenarios + +The complete catalog of E2E scenarios that prove Phase 1 works. Each scenario is implemented as an integration test in the sub-task(s) listed in **Owner**. The catalog as a whole is the release-readiness coverage check — if every row's tests pass, the project is functionally complete. + +These scenarios live in code (as integration tests). This list is the *registry* — the single place to ask "what does done look like?" + +#### Multi-key authentication + +| # | Scenario | Owner | +|---|---|---| +| 1 | Env with N SDK keys: every key authenticates downstream SDKs correctly; one upstream connection serves all. | T2.a, T3.c | +| 2 | Env with M mobile keys: every mobile key authenticates downstream SDKs correctly. | T1.c, T3.c | +| 3 | Mixed accepted set (SDK + mobile + env ID): all credentials route to the same env context. | T2.a, T3.c | + +#### Re-anchoring + +| # | Scenario | Owner | +|---|---|---| +| 4 | Voluntary anchor rotation: `sdkKey.value` changes; downstream SSE survives; events continue under the new anchor. | T2.c | +| 5 | Default expiry-driven rotation: backend marks current default as expiring and promotes a new one; relay re-anchors; downstream survives. | T2.c, T1.c | +| 6 | New-anchor client init fails: rollback; anchor pointer stays on old key; previous accepted set preserved; structured error logged + alarmed. | T2.c | +| 7 | Big-segment sync remains functional after re-anchor (recreated or re-wired per T2.d's choice). | T2.d | + +#### Key lifecycle + +| # | Scenario | Owner | +|---|---|---| +| 8 | Add a new key: joins the accepted set; existing downstream SDKs undisturbed. | T3.c | +| 9 | Graceful expiry: non-anchor key with `expiry` set; ticker drops it at the timestamp; *only that key's* downstream SDKs disconnect. | T1.c | +| 10 | Immediate revocation: key omitted from next RAC patch; reconcile drops it now; targeted disconnect. | T1.c, T3.c | +| 11 | De-expiry: existing entry's `expiry` removed in next payload; scheduled drop cancelled. | T3.c | +| 12 | Rename: array entry's `key` identifier changes while `value` is preserved; no credential disturbance; status endpoint reflects the new identifier. | T3.c, T4 | +| 13 | Mixed update (add + re-anchor + remove in a single payload): operations apply in order `add → re-anchor → remove`. | T3.c, T2.c | + +#### Defensive behavior + +| # | Scenario | Owner | +|---|---|---| +| 14 | Malformed RAC payload (`sdkKey.value` not present in `sdkKeys[]`): previous accepted set preserved; structured error + alarm logged; RAC stream disconnects and reconnects with jitter; subsequent fresh `put` from backend restores correct state. | T3.b, T3.c | + +#### Sources + +| # | Scenario | Owner | +|---|---|---| +| 15 | RAC multi-key path: scenarios 1–14 work end-to-end via RAC. | T3.c (RAC handler) | +| 16 | Offline archive multi-key path: scenarios 1–13 work end-to-end via filedata reload. | T3.c (offline handler) | + +#### Backward compatibility + +| # | Scenario | Owner | +|---|---|---| +| 17 | Single-key env behaves identically to v8 (the regression invariant — checked at every PR boundary, not just at release). | All sub-PRs; full test suite | +| 18 | Pre-Phase-1 v8 relay parses new-format payload gracefully (additive guarantee). | T3.a | +| 19 | Events payload schema preserved across Phase 1: every field identical to v8 except the credential. | T5.b | + +#### Observability + +| # | Scenario | Owner | +|---|---|---| +| 20 | Status endpoint: scalar fields = anchor (obscured); arrays = full accepted set; per-key `expiry` visible when present; entries stably ordered (anchor first, identifier-alphabetical). | T4 | +| 21 | Analytics events forwarded under the env's anchor key per kind, regardless of which accepted key the request came in on. | T2.c, T5.b | +| 22 | Diagnostic events proxy verbatim under the originating credential (deliberate asymmetry — preserved, not collapsed). | T2.c | + ### Release-readiness checklist Before merging `feat/concurrent-keys` to v8 (and again before deploying to production), run through: 1. All Wave 2 sub-tasks merged and tests passing. -2. End-to-end customer-journey integration tests pass (assembled from T5.a + per-task acceptance tests). +2. **Every scenario in "End-to-end acceptance scenarios" above passes** (the catalog is the explicit coverage check). 3. Events payload regression test (T5.b) passes against the full feature branch. -4. Single-key behavior verified identical to v8's baseline via full test suite. +4. Single-key behavior verified identical to v8's baseline via the full test suite. 5. Status endpoint manually inspected for both single-key and multi-key envs. -6. Defensive payload tests: malformed RAC payload → relay logs + preserves previous state. +6. Defensive payload tests: malformed RAC payload → relay logs + preserves previous state + reconnects RAC stream. This is a *checklist*, not a discrete task. Touched at release readiness, not as a separate sub-PR. From dda0f6375a4b59365bbd4ff540a8f5e3a6d17005 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Mon, 22 Jun 2026 08:14:50 -0700 Subject: [PATCH 13/66] test(integrationtests): increase SDK key expiry margin and big-segment sync timeout (#711) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduces flaky Integration Tests - staging failures by relaxing two timing constants only in tests—no production Relay behavior changes. In testSDKKeyExpires, the deprecated-key grace period is increased from 5s to 30s so sequential rotateSDKKeys calls (one per environment) still send a future expiry to staging after CI latency and clock skew; the test still sleeps only until keys expire before asserting cleanup. In verifyEvaluationWithBigSegment, the assert.Eventually poll window is increased from 20s to 60s so a second big-segment update can propagate via streaming in slow environments; passing runs still exit as soon as evaluations match. --- integrationtests/big_segments_test.go | 5 +++-- integrationtests/offline_mode_test.go | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/integrationtests/big_segments_test.go b/integrationtests/big_segments_test.go index 367acf58..37358317 100644 --- a/integrationtests/big_segments_test.go +++ b/integrationtests/big_segments_test.go @@ -110,7 +110,8 @@ func verifyEvaluationWithBigSegment( // Poll the evaluation endpoint until we see the expected flag values. We're using a // longer timeout here than we use in tests that don't involve big segments, because // the user segment state caching inside the SDK makes it hard to say how soon we'll - // see the effect of an update. + // see the effect of an update. 60s gives enough headroom for streaming propagation + // of a second segment update to complete in staging CI environments. success := assert.Eventually(t, func() bool { ok := true for i, env := range environments { @@ -125,7 +126,7 @@ func verifyEvaluationWithBigSegment( } } return ok - }, time.Second*20, time.Millisecond*100, "Did not see expected flag values from Relay") + }, time.Second*60, time.Millisecond*100, "Did not see expected flag values from Relay") if !success { manager.loggers.Infof("EXPLANATION OF TEST FAILURE FOLLOWS:") diff --git a/integrationtests/offline_mode_test.go b/integrationtests/offline_mode_test.go index a7b66f53..fb87f13f 100644 --- a/integrationtests/offline_mode_test.go +++ b/integrationtests/offline_mode_test.go @@ -168,7 +168,11 @@ func testSDKKeyExpires(t *testing.T, manager *integrationTestManager) { fileName := "archive.tar.gz" filePath := filepath.Join(manager.relaySharedDir, fileName) - const keyGracePeriod = 5 * time.Second + // 30s gives plenty of margin for multiple sequential staging API calls to succeed + // even with round-trip latency and clock skew between the CI runner and staging. + // Previously 5s was too tight: the second rotateSDKKey call could arrive at staging + // with an expiry at-or-before "now", causing a 400 Bad Request. + const keyGracePeriod = 30 * time.Second // Relay will check for expired keys at this interval. const cleanupInterval = 100 * time.Millisecond From 1cf348271577d20ca749eb003b3268e0c855622e Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 23 Jun 2026 13:19:58 -0700 Subject: [PATCH 14/66] test(relay): fix flaky TestConcurrentKeysHarnessReference SSE race (#717) Fixes intermittent failures in the RAC mock + SDK stream subtest of TestConcurrentKeysHarnessReference, where the SSE call could hit 503 before the background SDK client finished initializing. --- relay/concurrent_keys_harness_ref_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/relay/concurrent_keys_harness_ref_test.go b/relay/concurrent_keys_harness_ref_test.go index 072fcb42..448dcdde 100644 --- a/relay/concurrent_keys_harness_ref_test.go +++ b/relay/concurrent_keys_harness_ref_test.go @@ -120,7 +120,17 @@ func TestConcurrentKeysHarnessReference(t *testing.T) { // 3. Wait for the env to become available (confirms Relay processed the RAC put event). h := relayTestHelper{t: t, relay: relay} - _ = h.awaitEnvironment(harnessEnvID) + env := h.awaitEnvironment(harnessEnvID) + + // awaitEnvironment only waits until the env is discoverable by credential lookup. The SDK + // client is created in a background goroutine (go c.startSDKClient(...)), so GetClient() can + // still be nil at this point. Relay's stream middleware returns 503 (Service Unavailable) + // while GetClient() == nil, which would cause the SSE request below to fail intermittently. + // Wait for the client to be ready before connecting, mirroring the readiness poll in + // internal/relayenv/env_context_impl_test.go (TestChangeSDKKey). + require.Eventually(t, func() bool { + return env.GetClient() != nil + }, 5*time.Second, time.Millisecond*5, "timed out waiting for the SDK client to be ready") // 4. Connect to Relay's SSE stream and verify it serves a put event. req := sharedtest.BuildRequestWithAuth(http.MethodGet, "/all", harnessSDKKey, nil) From 4585b0b3c70e4254c9b2f47b51e46b3bb74a4987 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 23 Jun 2026 15:15:04 -0700 Subject: [PATCH 15/66] docs(concurrent-keys): add milestone view and record M2 task split Adds a delivery-oriented milestone view (M1-M5) mapping the existing tasks to demonstrable capabilities, and records the 2026-06-23 decision to finish Milestone 2 before heavy Milestone 3 by reducing T1.b (SDK-2538) to a behavior-neutral foundation. The handler wiring + UpdateCredential removal + payload validation move to T3.c (SDK-2547); the re-queue-on-primary-switch fixes (SDK + mobile) move to T2.c (SDK-2542). --- .agent-docs/concurrent-keys/phase1-plan.md | 31 ++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.agent-docs/concurrent-keys/phase1-plan.md b/.agent-docs/concurrent-keys/phase1-plan.md index a47d4a90..256c56c0 100644 --- a/.agent-docs/concurrent-keys/phase1-plan.md +++ b/.agent-docs/concurrent-keys/phase1-plan.md @@ -40,6 +40,31 @@ The "single-key behavior unchanged at every PR boundary" invariant is the load-b --- +## 2.1 Milestone view (delivery-oriented) + +§2's waves are about *sequencing*; this is about *demonstrable capability*. The work groups into five milestones; each is "done" when its end-to-end acceptance scenarios (§7) pass. + +| Milestone | Capability | Tickets | Status | +|---|---|---|---| +| **M1 — Parse the new payload** | Read `sdkKeys[]`/`mobileKeys[]` (+ per-key expiry); old relays ignore the new fields | SDK-2545 (T3.a) | ✅ Done (#702) | +| **M2 — Anchor + multi-key auth** | Accept N keys; one upstream connection via the anchor; every key authenticates downstream | SDK-2538 (foundation) + SDK-2546 (helper, #713) + SDK-2547 (wire handlers) | In progress | +| **M3 — Expiry & rotation** | Grace-period expiry/deprecation; re-anchoring; targeted downstream disconnects | SDK-2539 (ticker, #714) + SDK-2542 (re-anchor) + SDK-2577 (rotation leak cleanup) | Not started | +| **M4 — Big segments after re-anchor** | Big-segment sync survives a re-anchor | SDK-2543 (T2.d) | Not started | +| **M5 — Handler fan-out** | One handler per `(filter, provider)` instead of per credential | SDK-2544 (T2.e) | Not started | + +### Decision (2026-06-23): finish M2 before heavy M3; split T1.b into foundation + wiring + +To keep PRs reviewable and land a working multi-key milestone before taking on rotation, **T1.b (SDK-2538 / #712) is reduced to a behavior-neutral foundation**: + +- The `AcceptedSet` data model (incl. per-key expiry *fields* — the full payload is represented), the `Rotator.Reconcile` add / set-anchor / remove core, the anchor-only upstream client (absorbed from T2.a / SDK-2540), and the `ReconcileCredentials` API. +- It does **not** remove `UpdateCredential` and does **not** change the action handlers — so it carries zero production behavior change. Existing rotation keeps flowing through `UpdateCredential`. + +**M2 is completed by T3.b (SDK-2546 / #713) + T3.c (SDK-2547):** the helper builds the `AcceptedSet` from the full parsed key arrays, and T3.c wires both handlers to `ReconcileCredentials`, removes `UpdateCredential`, and validates the payload. There is no window where both credential paths are live — the handler switch and the `UpdateCredential` deletion land together in T3.c. + +**M3 owns the rotation behavior:** the grace-period deprecation, the cleanup ticker (SDK-2539), the robust re-anchor mechanism, and the "re-queue an already-accepted key when it becomes the primary/anchor" fixes (both SDK and mobile, SDK-2542). Re-anchoring (changing the anchor) is M3, not M2. + +--- + ## 3. Task list Each task has: ticket name, files touched, dependencies, estimates. Acceptance criteria live in the JIRA ticket; rationale lives in [`phase1-design.md`](./phase1-design.md). @@ -147,6 +172,8 @@ Reviewer-friendly comment to add at the top of the new fields: `// Consumed by T ### T1.b — `ReconcileCredentials` API +> **Scope reduced (2026-06-23) — see §2.1.** T1.b is now a behavior-neutral *foundation*: the `AcceptedSet` model (incl. per-key expiry fields), `Rotator.Reconcile` (add / set-anchor / remove), the anchor-only client, and the `ReconcileCredentials` API. It **keeps** `UpdateCredential` and does **not** touch the action handlers. The migration + removal described below moved to **T3.c (SDK-2547)**; the rotation refinements moved to T1.c (SDK-2539) / T2.c (SDK-2542). The original note is retained for context. + The new method replaces `UpdateCredential` *everywhere* — both call sites migrate in this same PR, and `UpdateCredential` + supporting types are removed. There are no external consumers to preserve. Today's API surface (to be removed): @@ -190,6 +217,8 @@ The switch case at `env_context_impl.go:448-463` currently calls `startSDKClient ### T2.c — Re-anchor mechanism +> **Added scope (2026-06-23):** T2.c also owns the "re-queue an already-accepted key when it becomes the primary" fixes from SDK-2538 / #712 review. **SDK:** when `Reconcile` moves the anchor onto an already-accepted non-anchor key, re-queue it as an addition so `addCredential` runs the anchor-only setup. **Mobile:** when the primary mobile key switches to an already-accepted mobile key, re-queue it so event forwarding follows, and gate `addCredential`'s mobile side-effect on the primary mobile key (Bugbot "Primary mobile switch skips setup", Medium). The SDK fix currently lives in #712 and moves here with the rotation work. + The big one. PoC findings (design §7 + [`phase1-T0-reanchor-poc-findings.md`](./phase1-T0-reanchor-poc-findings.md)) turned this from "TBD per PoC" into a concrete specification: 1. **Build** the new anchor's SDK client (do *not* flip the anchor pointer yet). @@ -240,6 +269,8 @@ A new helper (in `internal/envfactory/` or similar) that both `autoconfig_action ### T3.c — Wire both action handlers +> **This is the PR that completes Milestone 2 (2026-06-23) — see §2.1.** Beyond wiring the handlers, T3.c now also owns (moved from T1.b / SDK-2538): **removing `UpdateCredential` / `CredentialUpdate` and migrating both call sites**, and **building the `AcceptedSet` from the full parsed key arrays** (`params.AcceptedSDKKeys` / `AcceptedMobileKeys`, incl. per-key expiry) rather than the singular fields. Plus the **undefined/malformed-credential validation** below (referenced on #712 as "SDK-2534" — a mis-cite; it belongs here): catch undefined/empty credentials and a structurally-malformed payload (anchor `value` absent from `sdkKeys[]`) at parse/process time and surface a structured error instead of silently dropping. + Replace `UpdateCredential` calls with the new `ReconcileCredentials` API, via the shared helper. RAC handler and offline handler updates land in one PR (separate commits per Aaron's preference). **Malformed-payload handling** (design §9): when the shared helper signals a malformed payload, the RAC handler must (a) preserve the previous accepted set and (b) **disconnect and reconnect the RAC stream with jitter** to force a fresh `put` from the backend. The offline handler preserves state only (no equivalent reconnect since there's no live connection — wait for the next archive reload). From 2d42cf8b4e8db971b53076976a6f4228a691efd2 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 24 Jun 2026 12:05:13 -0700 Subject: [PATCH 16/66] feat(relayenv): add ReconcileCredentials API and AcceptedSet foundation (#712) Introduces AcceptedSet and AcceptedSetBuilder so an environment can describe a full desired credential state (multiple SDK/mobile keys with optional expiry, anchor SDK key, primary mobile key, env ID), with Build() rejecting empty sets and sets without a designated anchor. Adds Rotator.Reconcile and EnvContext.ReconcileCredentials to diff that desired set against current state, queue additions/expirations, and apply them via the existing triggerCredentialChanges path. PrimaryCredentials now reflects all accepted non-deprecated keys in the rotator maps; the legacy Rotate / RotateWithGrace path is updated to keep those maps aligned (including revoking the previous key when SDK grace is already expired). Anchor-only upstream behavior: in addCredential, only the anchor SDK key starts an upstream client and repoints SDK event/metrics forwarding; only the primary mobile key repoints mobile event forwarding. Extra accepted keys still get streams, handlers, and connection mappings. --- internal/credential/accepted_set.go | 78 ++++++++ internal/credential/accepted_set_builder.go | 112 +++++++++++ .../credential/accepted_set_builder_test.go | 54 +++++ internal/credential/accepted_set_test.go | 19 ++ internal/credential/rotator.go | 171 ++++++++++++++-- internal/credential/rotator_test.go | 188 ++++++++++++++++++ internal/relayenv/env_context.go | 13 ++ internal/relayenv/env_context_impl.go | 50 +++-- internal/relayenv/env_context_impl_test.go | 160 +++++++++++++-- .../relayenv/env_context_reanchor_test.go | 37 ++-- relay/autoconfig_actions.go | 1 + relay/autoconfig_actions_test.go | 58 ++++-- 12 files changed, 864 insertions(+), 77 deletions(-) create mode 100644 internal/credential/accepted_set.go create mode 100644 internal/credential/accepted_set_builder.go create mode 100644 internal/credential/accepted_set_builder_test.go create mode 100644 internal/credential/accepted_set_test.go diff --git a/internal/credential/accepted_set.go b/internal/credential/accepted_set.go new file mode 100644 index 00000000..c03fd038 --- /dev/null +++ b/internal/credential/accepted_set.go @@ -0,0 +1,78 @@ +package credential + +import ( + "errors" + "fmt" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" +) + +// AcceptedSet is the full set of credentials that an environment should accept after a reconcile. +// It carries every accepted server-side SDK key and mobile key — each with an optional per-key +// expiry — plus the single environment ID and two primary designations: +// +// - The anchor: the one SDK key that owns the environment's upstream connection. Set with +// WithPrimarySDKKey. +// - The primary mobile key: the singular default mobile key (the wire's mobKey), used where one +// mobile key is required, e.g. event forwarding. Set with WithPrimaryMobileKey. +// +// WithPrimarySDKKey / WithPrimaryMobileKey both add the key to the set and designate it, so adding a +// single key takes one call. Build requires that an anchor was designated. (Structural validation of +// the wire payload — undefined credentials, an anchor absent from the array — happens upstream when +// the payload is parsed into the set; see SDK-2547.) +// +// A key's expiry is taken from its entry in this set; the legacy sdkKey.expiring{} wire slot is not +// consulted when building it. +// +// Construct an AcceptedSet with AcceptedSetBuilder (see accepted_set_builder.go). +type AcceptedSet struct { + // sdkKeys and mobileKeys store each accepted key once, keyed by value, so duplicates collapse + // without a containment scan. The map value is the key's expiry: a nil *time.Time means the key + // is permanent. A nil map is a valid empty set (reads return absent; only the builder writes). + sdkKeys map[config.SDKKey]*time.Time + primarySdkKey config.SDKKey + mobileKeys map[config.MobileKey]*time.Time + primaryMobileKey config.MobileKey + envID config.EnvironmentID +} + +// hasSDKKey reports whether key is one of the set's accepted SDK keys. +func (s AcceptedSet) hasSDKKey(key config.SDKKey) bool { + _, ok := s.sdkKeys[key] + return ok +} + +// hasMobileKey reports whether key is one of the set's accepted mobile keys. +func (s AcceptedSet) hasMobileKey(key config.MobileKey) bool { + _, ok := s.mobileKeys[key] + return ok +} + +// errAcceptedSetMissingSDKKey is returned by AcceptedSetBuilder.Build when no SDK key was added. An +// environment must always have at least one SDK key (its anchor), so an empty set indicates a caller +// mistake rather than a benign edge case — surfacing it avoids a silent misconfiguration. +var errAcceptedSetMissingSDKKey = errors.New("accepted credential set must contain at least one SDK key") + +// MalformedCredentialSetError is returned by AcceptedSetBuilder.Build when the set's designated +// anchor SDK key is missing — a violation of the backend invariant that the anchor (sdkKey.value) +// always appears in sdkKeys[]. Validation happens once, at build time; Rotator.Reconcile trusts the +// set it is handed. +// +// Because Build rejects the set before it ever reaches Reconcile, the environment's previous accepted +// set is preserved on a malformed payload. The caller is responsible for the second half of the +// malformed-payload policy: reconnecting the RAC stream with jitter to force a fresh put. RAC is +// one-way push with no NAK channel, so without the reconnect the backend would believe the malformed +// patch was applied and would not send fresh state. +type MalformedCredentialSetError struct { + // Anchor is the anchor credential that was not found among the set's SDK keys. + Anchor SDKCredential +} + +func (e *MalformedCredentialSetError) Error() string { + if e.Anchor == nil { + return "malformed credential set: anchor SDK key is missing" + } + return fmt.Sprintf("malformed credential set: anchor SDK key %s is not present in the accepted set", + e.Anchor.Masked()) +} diff --git a/internal/credential/accepted_set_builder.go b/internal/credential/accepted_set_builder.go new file mode 100644 index 00000000..8e82189d --- /dev/null +++ b/internal/credential/accepted_set_builder.go @@ -0,0 +1,112 @@ +package credential + +import ( + "time" + + "github.com/launchdarkly/ld-relay/v8/config" +) + +// AcceptedSetBuilder accumulates the credentials for an AcceptedSet. Build validates the accumulated +// set (see Build) before returning it. +type AcceptedSetBuilder struct { + set AcceptedSet +} + +// NewAcceptedSetBuilder returns an empty AcceptedSetBuilder. +func NewAcceptedSetBuilder() *AcceptedSetBuilder { + return &AcceptedSetBuilder{ + set: AcceptedSet{ + sdkKeys: make(map[config.SDKKey]*time.Time), + mobileKeys: make(map[config.MobileKey]*time.Time), + }, + } +} + +// WithSDKKey adds a permanent (non-expiring) SDK key. It is a no-op if the key is undefined or +// already present. +func (b *AcceptedSetBuilder) WithSDKKey(key config.SDKKey) *AcceptedSetBuilder { + b.addSDKKey(key, nil) + return b +} + +// WithExpiringSDKKey adds an SDK key that should be accepted until the given expiry. It is a no-op +// if the key is undefined or already present. +func (b *AcceptedSetBuilder) WithExpiringSDKKey(key config.SDKKey, expiry time.Time) *AcceptedSetBuilder { + b.addSDKKey(key, &expiry) + return b +} + +// WithPrimarySDKKey adds key (if not already present) and designates it as the anchor — the SDK key +// that owns the environment's upstream connection. It is a no-op if the key is undefined. +func (b *AcceptedSetBuilder) WithPrimarySDKKey(key config.SDKKey) *AcceptedSetBuilder { + if key.Defined() { + b.addSDKKey(key, nil) + b.set.primarySdkKey = key + } + return b +} + +// addSDKKey records the key with the given expiry (nil = permanent), skipping undefined keys and +// keys already in the set (the first expiry recorded for a key wins). +func (b *AcceptedSetBuilder) addSDKKey(key config.SDKKey, expiry *time.Time) { + if !key.Defined() || b.set.hasSDKKey(key) { + return + } + b.set.sdkKeys[key] = expiry +} + +// WithMobileKey adds a permanent (non-expiring) mobile key. It is a no-op if the key is undefined or +// already present. +func (b *AcceptedSetBuilder) WithMobileKey(key config.MobileKey) *AcceptedSetBuilder { + b.addMobileKey(key, nil) + return b +} + +// WithExpiringMobileKey adds a mobile key that should be accepted until the given expiry. It is a +// no-op if the key is undefined or already present. +func (b *AcceptedSetBuilder) WithExpiringMobileKey(key config.MobileKey, expiry time.Time) *AcceptedSetBuilder { + b.addMobileKey(key, &expiry) + return b +} + +// WithPrimaryMobileKey adds key (if not already present) and designates it as the primary mobile +// key — the singular default (the wire's mobKey) used where one mobile key is required, e.g. event +// forwarding. It is a no-op if the key is undefined. +func (b *AcceptedSetBuilder) WithPrimaryMobileKey(key config.MobileKey) *AcceptedSetBuilder { + if key.Defined() { + b.addMobileKey(key, nil) + b.set.primaryMobileKey = key + } + return b +} + +// addMobileKey records the key with the given expiry (nil = permanent), skipping undefined keys and +// keys already in the set (the first expiry recorded for a key wins). +func (b *AcceptedSetBuilder) addMobileKey(key config.MobileKey, expiry *time.Time) { + if !key.Defined() || b.set.hasMobileKey(key) { + return + } + b.set.mobileKeys[key] = expiry +} + +// WithEnvironmentID sets the environment ID. It is a no-op if the ID is undefined. +func (b *AcceptedSetBuilder) WithEnvironmentID(id config.EnvironmentID) *AcceptedSetBuilder { + if id.Defined() { + b.set.envID = id + } + return b +} + +// Build validates and returns the accumulated AcceptedSet. It returns errAcceptedSetMissingSDKKey if +// no SDK key was added, or a *MalformedCredentialSetError if no anchor was designated (via +// WithPrimarySDKKey). Because WithPrimarySDKKey also adds the key, a designated anchor is always +// among the accepted SDK keys. +func (b *AcceptedSetBuilder) Build() (AcceptedSet, error) { + if len(b.set.sdkKeys) == 0 { + return AcceptedSet{}, errAcceptedSetMissingSDKKey + } + if !b.set.primarySdkKey.Defined() { + return AcceptedSet{}, &MalformedCredentialSetError{Anchor: nil} + } + return b.set, nil +} diff --git a/internal/credential/accepted_set_builder_test.go b/internal/credential/accepted_set_builder_test.go new file mode 100644 index 00000000..b6e92d91 --- /dev/null +++ b/internal/credential/accepted_set_builder_test.go @@ -0,0 +1,54 @@ +package credential + +import ( + "testing" + + "github.com/launchdarkly/ld-relay/v8/config" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAcceptedSetBuilderValidation(t *testing.T) { + // No SDK key at all is a caller error. + _, err := NewAcceptedSetBuilder(). + WithMobileKey(config.MobileKey("mob")). + WithEnvironmentID(config.EnvironmentID("env")). + Build() + require.ErrorIs(t, err, errAcceptedSetMissingSDKKey) + + // An SDK key with no designated anchor is malformed. + var malformed *MalformedCredentialSetError + _, err = NewAcceptedSetBuilder().WithSDKKey(config.SDKKey("sdk")).Build() + require.ErrorAs(t, err, &malformed) + + // WithPrimarySDKKey adds the key and designates it as the anchor, so Build succeeds. + set, err := NewAcceptedSetBuilder().WithPrimarySDKKey(config.SDKKey("sdk")).Build() + require.NoError(t, err) + assert.True(t, set.hasSDKKey(config.SDKKey("sdk"))) + assert.Equal(t, config.SDKKey("sdk"), set.primarySdkKey) +} + +func TestAcceptedSetBuilderDeduplicates(t *testing.T) { + // Adding the same key more than once (including via WithPrimary*) keeps a single entry. + set := mustBuild(t, NewAcceptedSetBuilder(). + WithSDKKey(config.SDKKey("sdk")). + WithPrimarySDKKey(config.SDKKey("sdk")). + WithSDKKey(config.SDKKey("sdk")). + WithMobileKey(config.MobileKey("mob")). + WithPrimaryMobileKey(config.MobileKey("mob"))) + + assert.Len(t, set.sdkKeys, 1) + assert.Len(t, set.mobileKeys, 1) + assert.Equal(t, config.SDKKey("sdk"), set.primarySdkKey) + assert.Equal(t, config.MobileKey("mob"), set.primaryMobileKey) +} + +// mustBuild builds the set and fails the test if validation rejects it. It is shared by the builder +// tests and the Reconcile tests in rotator_test.go. +func mustBuild(t *testing.T, b *AcceptedSetBuilder) AcceptedSet { + t.Helper() + set, err := b.Build() + require.NoError(t, err) + return set +} diff --git a/internal/credential/accepted_set_test.go b/internal/credential/accepted_set_test.go new file mode 100644 index 00000000..dec56d34 --- /dev/null +++ b/internal/credential/accepted_set_test.go @@ -0,0 +1,19 @@ +package credential + +import ( + "testing" + + "github.com/launchdarkly/ld-relay/v8/config" + + "github.com/stretchr/testify/assert" +) + +func TestMalformedCredentialSetErrorMessage(t *testing.T) { + // A nil anchor reports "missing" rather than dereferencing a nil credential. + assert.Equal(t, "malformed credential set: anchor SDK key is missing", + (&MalformedCredentialSetError{Anchor: nil}).Error()) + + // A defined anchor is masked in the message. + assert.Contains(t, (&MalformedCredentialSetError{Anchor: config.SDKKey("sdk-abcd1234")}).Error(), + "...1234") +} diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 536a63fe..1305b120 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -1,7 +1,6 @@ package credential import ( - "slices" "sync" "time" @@ -11,7 +10,7 @@ import ( // acceptedKeyInfo holds per-key metadata for the accepted-set maps. type acceptedKeyInfo struct { - expiry *time.Time //nolint:unused // nil = permanent; read by T1.b (ReconcileCredentials) + expiry *time.Time // nil = permanent } type Rotator struct { @@ -22,7 +21,7 @@ type Rotator struct { // deprecatedMobileKeys stores mobile keys being phased out with a grace period, keyed // by credential value with the associated expiry time. StepTime does not yet act on - // these entries — that is deferred to T1.c (generalize cleanup ticker). + // these entries; generalizing the cleanup ticker to drop them is handled separately. deprecatedMobileKeys map[config.MobileKey]time.Time // There is only one environment ID active at a given time, and it won't actually be rotated. The mechanism is @@ -116,14 +115,29 @@ func (r *Rotator) PrimaryCredentials() []SDKCredential { return r.primaryCredentials() } +// primaryCredentials returns every accepted, non-deprecated credential: all accepted SDK keys, all +// accepted mobile keys, and the environment ID. The primary SDK key and primary mobile key are always +// present in the accepted-set maps (maintained by Initialize, the legacy rotation path, and Reconcile) +// and are never left marked deprecated, so a plain pass over the maps already includes them. func (r *Rotator) primaryCredentials() []SDKCredential { - return slices.DeleteFunc([]SDKCredential{ - r.primarySdkKey, - r.primaryMobileKey, - r.primaryEnvironmentID, - }, func(cred SDKCredential) bool { - return !cred.Defined() - }) + creds := make([]SDKCredential, 0, len(r.acceptedSDKKeys)+len(r.acceptedMobileKeys)+1) + + for key := range r.acceptedSDKKeys { + if _, deprecated := r.deprecatedSdkKeys[key]; deprecated { + continue + } + creds = append(creds, key) + } + for key := range r.acceptedMobileKeys { + if _, deprecated := r.deprecatedMobileKeys[key]; deprecated { + continue + } + creds = append(creds, key) + } + if r.primaryEnvironmentID.Defined() { + creds = append(creds, r.primaryEnvironmentID) + } + return creds } func (r *Rotator) deprecatedCredentials() []SDKCredential { @@ -212,7 +226,7 @@ func (r *Rotator) updateEnvironmentID(envID config.EnvironmentID) { // updateMobileKey sets a new primary mobile key. When grace is nil the outgoing key is // immediately revoked; when non-nil its expiry is stored in deprecatedMobileKeys. -// StepTime does not yet act on deprecatedMobileKeys — that is deferred to T1.c. +// StepTime does not yet act on deprecatedMobileKeys; that cleanup is handled separately. func (r *Rotator) updateMobileKey(mobileKey config.MobileKey, grace *GracePeriod) { r.mu.Lock() defer r.mu.Unlock() @@ -221,6 +235,11 @@ func (r *Rotator) updateMobileKey(mobileKey config.MobileKey, grace *GracePeriod } previous := r.primaryMobileKey r.primaryMobileKey = mobileKey + // Keep the accepted-set map (the source of truth for PrimaryCredentials) consistent with the + // legacy rotation path. + if _, ok := r.acceptedMobileKeys[mobileKey]; !ok { + r.acceptedMobileKeys[mobileKey] = &acceptedKeyInfo{} + } delete(r.deprecatedMobileKeys, mobileKey) r.additions = append(r.additions, mobileKey) if !previous.Defined() { @@ -228,11 +247,13 @@ func (r *Rotator) updateMobileKey(mobileKey config.MobileKey, grace *GracePeriod return } if grace == nil { + delete(r.acceptedMobileKeys, previous) r.expirations = append(r.expirations, previous) r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", previous.Masked(), mobileKey.Masked()) return } if grace.Expired() { + delete(r.acceptedMobileKeys, previous) r.loggers.Infof("Deprecated mobile key %s already expired at %v; revoking immediately", previous.Masked(), grace.expiry) r.expirations = append(r.expirations, previous) return @@ -249,6 +270,13 @@ func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { } previous := r.primarySdkKey r.primarySdkKey = newKey + // Keep the accepted-set map (the source of truth for PrimaryCredentials) consistent: the new + // primary is accepted and is no longer deprecated, even if it was being phased out before. Mirrors + // updateMobileKey for mobile keys. + if _, ok := r.acceptedSDKKeys[newKey]; !ok { + r.acceptedSDKKeys[newKey] = &acceptedKeyInfo{} + } + delete(r.deprecatedSdkKeys, newKey) r.additions = append(r.additions, newKey) r.loggers.Infof("New primary SDK key is %s", newKey.Masked()) @@ -257,6 +285,7 @@ func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { func (r *Rotator) immediatelyRevoke(key config.SDKKey) { if key.Defined() { + delete(r.acceptedSDKKeys, key) r.expirations = append(r.expirations, key) r.loggers.Infof("SDK key %s has been immediately revoked", key.Masked()) } @@ -291,7 +320,8 @@ func (r *Rotator) updateSDKKey(sdkKey config.SDKKey, grace *GracePeriod) { } if grace.Expired() { - r.loggers.Infof("Deprecated SDK key %s already expired at %v; ignoring", grace.key.Masked(), grace.expiry) + r.loggers.Infof("Deprecated SDK key %s already expired at %v; revoking the previous key immediately", grace.key.Masked(), grace.expiry) + r.immediatelyRevoke(previous) return } @@ -307,6 +337,7 @@ func (r *Rotator) updateSDKKey(sdkKey config.SDKKey, grace *GracePeriod) { func (r *Rotator) expireSDKKey(sdkKey config.SDKKey) { r.loggers.Infof("Deprecated SDK key %s has expired and is no longer valid for authentication", sdkKey.Masked()) delete(r.deprecatedSdkKeys, sdkKey) + delete(r.acceptedSDKKeys, sdkKey) r.expirations = append(r.expirations, sdkKey) } @@ -327,3 +358,119 @@ func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expiration r.expirations = nil return } + +// Reconcile updates the rotator to match set. The set names its own anchor (the primary SDK key) and +// primary mobile key. It diffs the desired accepted set against the current one and queues additions +// and expirations (drained by the next StepTime call); keys newly present are accepted, and keys no +// longer present are revoked. Per-key expiry is stored as data on the accepted entry — grace-period +// deprecation and the cleanup ticker that drops expiring keys are handled separately. An undefined +// environment ID leaves the current one unchanged, since environments are removed via teardown +// rather than reconcile. +// +// The set is assumed well-formed: AcceptedSetBuilder.Build validates that an anchor was designated +// (and, because WithPrimarySDKKey adds the key as it designates it, that the anchor is among the SDK +// keys), so Reconcile trusts what it is handed rather than re-validating. +func (r *Rotator) Reconcile(set AcceptedSet, now time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + + r.reconcileSDKKeys(set, set.primarySdkKey, now) + r.reconcileMobileKeys(set, now) + r.reconcileEnvironmentID(set) +} + +// reconcilableKey constrains the generic reconcile helper to a comparable credential (so it can key a +// map) that is also an SDKCredential (so it can be logged and appended to the credential lists). +type reconcilableKey interface { + comparable + SDKCredential +} + +// reconcileAcceptedKeys diffs the desired keys against the currently-accepted ones (SDK or mobile, +// same algorithm): a desired key not yet accepted is recorded and queued as an addition; an accepted +// key no longer desired is dropped and queued as an expiration. Either way the key is cleared from the +// deprecated map — a key the set accepts is not deprecated, and a key it revokes is gone. Per-key +// expiry is stored as data on the accepted entry; the cleanup ticker is what later acts on it. The +// caller must hold the write lock. +func reconcileAcceptedKeys[K reconcilableKey]( + desired map[K]*time.Time, + accepted map[K]*acceptedKeyInfo, + deprecated map[K]time.Time, + additions *[]SDKCredential, + expirations *[]SDKCredential, + loggers ldlog.Loggers, + kind string, +) { + // First pass: walk every key the set wants us to accept. If we already accept it, just refresh + // its expiry; if it's new, start accepting it and queue it as an addition. Either way, a desired + // key can't also be deprecated, so clear any stale deprecation for it. + for key, expiry := range desired { + if info, ok := accepted[key]; ok { + info.expiry = expiry + } else { + accepted[key] = &acceptedKeyInfo{expiry: expiry} + *additions = append(*additions, key) + loggers.Infof("%s %s is now accepted", kind, key.Masked()) + } + delete(deprecated, key) + } + // Second pass: walk every key we currently accept and drop the ones the set no longer wants. + // Keys still desired were handled above, so skip them; the rest are revoked outright (removed + // from both maps) and queued as expirations. + for key := range accepted { + if _, ok := desired[key]; ok { + continue + } + delete(accepted, key) + delete(deprecated, key) + *expirations = append(*expirations, key) + loggers.Infof("%s %s is no longer accepted and has been revoked", kind, key.Masked()) + } +} + +// reconcileSDKKeys diffs the desired SDK keys against the accepted set and applies the result via +// reconcileAcceptedKeys. The anchor is always accepted and permanent, regardless of any expiry the +// payload may carry for it. The caller must hold the write lock. +func (r *Rotator) reconcileSDKKeys(set AcceptedSet, anchor config.SDKKey, now time.Time) { + desired := make(map[config.SDKKey]*time.Time, len(set.sdkKeys)) + for key, expiry := range set.sdkKeys { + if expiry != nil && !now.Before(*expiry) { + continue // already expired; treat as absent + } + desired[key] = expiry + } + desired[anchor] = nil + reconcileAcceptedKeys(desired, r.acceptedSDKKeys, r.deprecatedSdkKeys, &r.additions, &r.expirations, r.loggers, "SDK key") + r.primarySdkKey = anchor +} + +// reconcileMobileKeys mirrors reconcileSDKKeys for mobile keys. The primary mobile key — the wire's +// singular mobKey, used where one mobile key is required (e.g. event forwarding) — is always accepted +// and permanent; an empty value means the set declared no mobile key. The caller must hold the lock. +func (r *Rotator) reconcileMobileKeys(set AcceptedSet, now time.Time) { + desired := make(map[config.MobileKey]*time.Time, len(set.mobileKeys)) + for key, expiry := range set.mobileKeys { + if expiry != nil && !now.Before(*expiry) { + continue // already expired; treat as absent + } + desired[key] = expiry + } + if set.primaryMobileKey.Defined() { + desired[set.primaryMobileKey] = nil + } + reconcileAcceptedKeys(desired, r.acceptedMobileKeys, r.deprecatedMobileKeys, &r.additions, &r.expirations, r.loggers, "Mobile key") + r.primaryMobileKey = set.primaryMobileKey +} + +// reconcileEnvironmentID updates the environment ID if the set carries a new one. The caller must +// hold the write lock. +func (r *Rotator) reconcileEnvironmentID(set AcceptedSet) { + if !set.envID.Defined() || set.envID == r.primaryEnvironmentID { + return + } + if r.primaryEnvironmentID.Defined() { + r.expirations = append(r.expirations, r.primaryEnvironmentID) + } + r.primaryEnvironmentID = set.envID + r.additions = append(r.additions, set.envID) +} diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 86c0a346..510e9cd5 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -8,8 +8,13 @@ import ( "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" "github.com/launchdarkly/ld-relay/v8/config" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func newTestRotator() *Rotator { + return NewRotator(ldlogtest.NewMockLog().Loggers) +} + func TestNewRotator(t *testing.T) { mockLog := ldlogtest.NewMockLog() rotator := NewRotator(mockLog.Loggers) @@ -307,6 +312,8 @@ func TestRotateWithGraceMobileKey(t *testing.T) { additions, expirations := rotator.StepTime(now) assert.ElementsMatch(t, []SDKCredential{mob2}, additions) assert.ElementsMatch(t, []SDKCredential{mob1}, expirations) + // The immediately-revoked key must leave the accepted set, not linger in PrimaryCredentials. + assert.NotContains(t, rotator.PrimaryCredentials(), SDKCredential(mob1)) }) t.Run("immediately revokes outgoing key when grace is nil", func(t *testing.T) { @@ -324,6 +331,8 @@ func TestRotateWithGraceMobileKey(t *testing.T) { additions, expirations := rotator.StepTime(time.Now()) assert.ElementsMatch(t, []SDKCredential{mob2}, additions) assert.ElementsMatch(t, []SDKCredential{mob1}, expirations) + // The immediately-revoked key must leave the accepted set, not linger in PrimaryCredentials. + assert.NotContains(t, rotator.PrimaryCredentials(), SDKCredential(mob1)) }) t.Run("re-promoting a deprecated key removes it from the deprecated set", func(t *testing.T) { @@ -353,3 +362,182 @@ func TestRotateWithGraceMobileKey(t *testing.T) { assert.ElementsMatch(t, []SDKCredential{mob2}, expirations) }) } + +func TestRotateSDKKeyRePromoteClearsDeprecation(t *testing.T) { + // Re-promoting a deprecated SDK key back to primary must clear its deprecated mark, so + // PrimaryCredentials lists it (mirrors the mobile re-promote behavior). + rotator := newTestRotator() + key1 := config.SDKKey("key1") + key2 := config.SDKKey("key2") + start := time.Unix(10000, 0) + + rotator.Initialize([]SDKCredential{key1}) + rotator.RotateWithGrace(key2, NewGracePeriod(key1, start.Add(time.Hour), start)) // deprecate key1 + rotator.StepTime(start) + assert.ElementsMatch(t, []SDKCredential{key1}, rotator.DeprecatedCredentials()) + + rotator.RotateWithGrace(key1, nil) // re-promote key1 + rotator.StepTime(start) + + assert.Equal(t, key1, rotator.SDKKey()) + assert.Contains(t, rotator.PrimaryCredentials(), SDKCredential(key1)) + assert.NotContains(t, rotator.DeprecatedCredentials(), SDKCredential(key1)) +} + +func TestRotateSDKKeyWithExpiredGraceRevokesPrevious(t *testing.T) { + // A legacy SDK rotation whose grace period is already expired must revoke the swapped-out key, + // not leave it enabled alongside the new anchor (mirrors updateMobileKey). + rotator := newTestRotator() + key1 := config.SDKKey("key1") + key2 := config.SDKKey("key2") + expiry := time.Unix(10000, 0) + now := expiry.Add(time.Hour) // now is after expiry + + rotator.Initialize([]SDKCredential{key1}) + rotator.RotateWithGrace(key2, NewGracePeriod(key1, expiry, now)) + + assert.Equal(t, key2, rotator.SDKKey()) + additions, expirations := rotator.StepTime(now) + assert.ElementsMatch(t, []SDKCredential{key2}, additions) + assert.ElementsMatch(t, []SDKCredential{key1}, expirations) + assert.NotContains(t, rotator.PrimaryCredentials(), SDKCredential(key1)) +} + +func TestReconcileAnchorOnly(t *testing.T) { + r := newTestRotator() + anchor := config.SDKKey("anchor") + now := time.Now() + + r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor)), now) + additions, expirations := r.StepTime(now) + + assert.ElementsMatch(t, []SDKCredential{anchor}, additions) + assert.Empty(t, expirations) + assert.Equal(t, anchor, r.SDKKey()) + assert.ElementsMatch(t, []SDKCredential{anchor}, r.PrimaryCredentials()) + assert.Empty(t, r.DeprecatedCredentials()) +} + +func TestReconcileMultipleSDKKeys(t *testing.T) { + r := newTestRotator() + anchor := config.SDKKey("anchor") + other := config.SDKKey("other") + now := time.Now() + + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor).WithSDKKey(other)), now) + additions, expirations := r.StepTime(now) + + // Both server keys are accepted; only the anchor is primary. + assert.ElementsMatch(t, []SDKCredential{anchor, other}, additions) + assert.Empty(t, expirations) + assert.Equal(t, anchor, r.SDKKey()) + assert.ElementsMatch(t, []SDKCredential{anchor, other}, r.PrimaryCredentials()) + assert.Empty(t, r.DeprecatedCredentials()) +} + +func TestReconcileMultipleMobileKeys(t *testing.T) { + r := newTestRotator() + anchor := config.SDKKey("anchor") + mob1 := config.MobileKey("mob1") + mob2 := config.MobileKey("mob2") + now := time.Now() + + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor).WithPrimaryMobileKey(mob1).WithMobileKey(mob2)), now) + additions, _ := r.StepTime(now) + + // Every mobile key is accepted; the designated one is the primary. + assert.ElementsMatch(t, []SDKCredential{anchor, mob1, mob2}, additions) + assert.Equal(t, mob1, r.MobileKey()) + assert.ElementsMatch(t, []SDKCredential{anchor, mob1, mob2}, r.PrimaryCredentials()) +} + +func TestReconcileRevokesOmittedKeys(t *testing.T) { + r := newTestRotator() + anchor := config.SDKKey("anchor") + other := config.SDKKey("other") + mob := config.MobileKey("mob") + now := time.Now() + + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor).WithSDKKey(other).WithPrimaryMobileKey(mob)), now) + r.StepTime(now) + + // Reconciling to just the anchor revokes the omitted server and mobile keys. + r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor)), now) + additions, expirations := r.StepTime(now) + + assert.Empty(t, additions) + assert.ElementsMatch(t, []SDKCredential{other, mob}, expirations) + assert.ElementsMatch(t, []SDKCredential{anchor}, r.PrimaryCredentials()) +} + +func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { + // The foundation stores per-key expiry but does not yet act on it (no grace-period deprecation, + // no cleanup ticker — those are handled separately). An expiring key is simply accepted. + r := newTestRotator() + anchor := config.SDKKey("anchor") + expiringSDK := config.SDKKey("expiring-sdk") + mob := config.MobileKey("mob") + expiringMobile := config.MobileKey("expiring-mob") + now := time.Unix(1000, 0) + + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithPrimarySDKKey(anchor). + WithExpiringSDKKey(expiringSDK, now.Add(time.Hour)). + WithPrimaryMobileKey(mob). + WithExpiringMobileKey(expiringMobile, now.Add(time.Hour))), + now) + additions, expirations := r.StepTime(now) + + assert.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, additions) + assert.Empty(t, expirations) + // All keys are accepted and non-deprecated in the foundation. + assert.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, r.PrimaryCredentials()) + assert.Empty(t, r.DeprecatedCredentials()) +} + +func TestReconcilePrimaryMobileKeyIsAlwaysAccepted(t *testing.T) { + // Defensive: even if the designated primary mobile key is also listed with a past expiry, it must + // stay accepted (mirroring the SDK anchor), so PrimaryCredentials never reports a torn-down key. + r := newTestRotator() + anchor := config.SDKKey("anchor") + mob := config.MobileKey("mob") + now := time.Unix(1000, 0) + + set := mustBuild(t, NewAcceptedSetBuilder(). + WithPrimarySDKKey(anchor). + WithExpiringMobileKey(mob, now.Add(-time.Hour)). // already expired in the payload... + WithPrimaryMobileKey(mob)) // ...but designated as the primary + r.Reconcile(set, now) + r.StepTime(now) + + assert.Equal(t, mob, r.MobileKey()) + assert.Contains(t, r.PrimaryCredentials(), SDKCredential(mob)) + _, accepted := r.acceptedMobileKeys[mob] + assert.True(t, accepted, "the primary mobile key must remain in the accepted set") +} + +func TestReconcileClearsStaleDeprecationForAcceptedKey(t *testing.T) { + // A key left in the deprecated set by the legacy rotation path must be treated as fully accepted + // once a reconcile includes it, not silently skipped by PrimaryCredentials. + r := newTestRotator() + old := config.SDKKey("old") + anchor := config.SDKKey("anchor") + now := time.Unix(1000, 0) + + r.Initialize([]SDKCredential{old}) + r.RotateWithGrace(anchor, NewGracePeriod(old, now.Add(time.Hour), now)) // deprecate `old` with grace + r.StepTime(now) + require.ElementsMatch(t, []SDKCredential{old}, r.DeprecatedCredentials()) + + // Reconcile to a set that fully accepts both keys. + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor).WithSDKKey(old)), now) + r.StepTime(now) + + assert.Contains(t, r.PrimaryCredentials(), SDKCredential(old)) + assert.Empty(t, r.DeprecatedCredentials()) +} diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index 7f5dfd13..fbfcd357 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -79,8 +79,21 @@ type EnvContext interface { // UpdateCredential updates the environment with a new credential, optionally deprecating a previous one // with a grace period. + // + // This is the legacy single-credential rotation API. It is retained while the action handlers still + // drive rotation through it; the full-set ReconcileCredentials below will take over once the handlers + // are migrated to it. UpdateCredential(update *CredentialUpdate) + // ReconcileCredentials atomically reconciles the environment's accepted credentials to match + // newSet. The set names its own anchor (the SDK key that owns the upstream connection) and + // primary mobile key. The method owns the order of operations internally (add → re-anchor → + // remove); callers do not sequence. + // + // newSet is assumed well-formed: it is built and validated via credential.AcceptedSetBuilder + // (which guarantees an anchor) before reaching here, so this method does not re-validate. + ReconcileCredentials(newSet credential.AcceptedSet) + // GetCredentials returns all currently enabled and non-deprecated credentials for the environment. GetCredentials() []credential.SDKCredential diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 1e56fa0c..1b586af3 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -437,8 +437,11 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { } // A new SDK key means: - // 1. we should start a new SDK client* - // 2. we should tell all event forwarding components that use an SDK key to use the new one. + // 1. we should start a new SDK client*, but only for the anchor: there is a single upstream + // connection per environment, owned by the anchor key. Non-anchor server keys get envStreams + // + handler bundles above, but no upstream client — matching today's mobile-key behavior. + // 2. we should tell all event forwarding components that use an SDK key to use the new one, + // again only when it is the anchor, since events collapse to the anchor per kind. // A new mobile key does not require starting a new SDK client, but does requiring updating any event forwarding // components that use a mobile key. // *Note: we only start a new SDK client in online mode. This is somewhat of an architectural hack because EnvContextImpl @@ -447,18 +450,25 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { // So, the effect in offline mode when adding/removing credentials is just setting up the new credential mappings. switch key := newCredential.(type) { case config.SDKKey: - if !c.offline { - go c.startSDKClient(key, nil, false) - } - if c.metricsEventPub != nil { // metrics event publisher always uses SDK key - c.metricsEventPub.ReplaceCredential(key) - } - if c.eventDispatcher != nil { - c.eventDispatcher.ReplaceCredential(key) + if key == c.keyRotator.SDKKey() { + if !c.offline { + go c.startSDKClient(key, nil, false) + } + if c.metricsEventPub != nil { // metrics event publisher always uses SDK key + c.metricsEventPub.ReplaceCredential(key) + } + if c.eventDispatcher != nil { + c.eventDispatcher.ReplaceCredential(key) + } } case config.MobileKey: - if c.eventDispatcher != nil { - c.eventDispatcher.ReplaceCredential(key) + // Mobile-key event forwarding collapses to the primary mobile key, mirroring the anchor-only + // behavior for SDK keys above: only the primary mobile key repoints the event dispatcher, so a + // non-primary mobile key accepted in the same reconcile does not steal event forwarding. + if key == c.keyRotator.MobileKey() { + if c.eventDispatcher != nil { + c.eventDispatcher.ReplaceCredential(key) + } } } @@ -559,6 +569,22 @@ func (c *envContextImpl) UpdateCredential(update *CredentialUpdate) { c.triggerCredentialChanges(update.now) } +func (c *envContextImpl) ReconcileCredentials(newSet credential.AcceptedSet) { + c.reconcileCredentials(newSet, time.Now()) +} + +// reconcileCredentials is the time-injectable implementation of ReconcileCredentials. now is the +// reference time for expiry math; production callers pass time.Now() via ReconcileCredentials. +// +// The Rotator owns the diff (add → re-anchor → remove) and queues the resulting additions and +// expirations; triggerCredentialChanges then applies them, draining additions before expirations so +// the accepted set is a superset during the transition. addCredential opens an upstream client only +// for the anchor, so non-anchor server keys are accepted and routed without a second connection. +func (c *envContextImpl) reconcileCredentials(newSet credential.AcceptedSet, now time.Time) { + c.keyRotator.Reconcile(newSet, now) + c.triggerCredentialChanges(now) +} + func (c *envContextImpl) triggerCredentialChanges(now time.Time) { additions, expirations := c.keyRotator.StepTime(now) for _, cred := range additions { diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 5c0aa671..8915e581 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -178,6 +178,14 @@ func TestLogPrefix(t *testing.T) { testPrefix("impossibly short env ID", LogNameIsEnvID, config.SDKKey("1234567890"), config.EnvironmentID("hij"), "[env: hij]") } +// mustBuildAcceptedSet builds the set from b, failing the test if Build returns an error. +func mustBuildAcceptedSet(t *testing.T, b *credential.AcceptedSetBuilder) credential.AcceptedSet { + t.Helper() + set, err := b.Build() + require.NoError(t, err) + return set +} + func TestAddRemoveCredential(t *testing.T) { envConfig := st.EnvMain.Config @@ -189,22 +197,30 @@ func TestAddRemoveCredential(t *testing.T) { assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) - env.UpdateCredential(NewCredentialUpdate(st.EnvWithAllCredentials.Config.MobileKey)) - env.UpdateCredential(NewCredentialUpdate(st.EnvWithAllCredentials.Config.EnvID)) + mobileKey := st.EnvWithAllCredentials.Config.MobileKey + envID := st.EnvWithAllCredentials.Config.EnvID + + // Reconcile to the full set: the SDK key (anchor) plus a mobile key and an environment ID. + env.ReconcileCredentials( + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithPrimarySDKKey(envConfig.SDKKey).WithPrimaryMobileKey(mobileKey).WithEnvironmentID(envID))) creds := env.GetCredentials() assert.Len(t, creds, 3) assert.Contains(t, creds, envConfig.SDKKey) - assert.Contains(t, creds, st.EnvWithAllCredentials.Config.MobileKey) - assert.Contains(t, creds, st.EnvWithAllCredentials.Config.EnvID) + assert.Contains(t, creds, mobileKey) + assert.Contains(t, creds, envID) - env.UpdateCredential(NewCredentialUpdate(config.MobileKey("evict-the-previous-key"))) + // Reconciling with a different mobile key evicts the previous one. + newMobileKey := config.MobileKey("evict-the-previous-key") + env.ReconcileCredentials( + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithPrimarySDKKey(envConfig.SDKKey).WithPrimaryMobileKey(newMobileKey).WithEnvironmentID(envID))) creds = env.GetCredentials() assert.Len(t, creds, 3) assert.Contains(t, creds, envConfig.SDKKey) - assert.NotContains(t, creds, st.EnvWithAllCredentials.Config.MobileKey) - assert.Contains(t, creds, st.EnvWithAllCredentials.Config.EnvID) + assert.NotContains(t, creds, mobileKey) + assert.Contains(t, creds, newMobileKey) + assert.Contains(t, creds, envID) } func TestAddExistingCredentialDoesNothing(t *testing.T) { @@ -218,19 +234,23 @@ func TestAddExistingCredentialDoesNothing(t *testing.T) { assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) - env.UpdateCredential(NewCredentialUpdate(st.EnvWithAllCredentials.Config.MobileKey)) + mobileKey := st.EnvWithAllCredentials.Config.MobileKey + set := mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithPrimarySDKKey(envConfig.SDKKey).WithPrimaryMobileKey(mobileKey)) + + env.ReconcileCredentials(set) creds := env.GetCredentials() assert.Len(t, creds, 2) assert.Contains(t, creds, envConfig.SDKKey) - assert.Contains(t, creds, st.EnvWithAllCredentials.Config.MobileKey) + assert.Contains(t, creds, mobileKey) - env.UpdateCredential(NewCredentialUpdate(st.EnvWithAllCredentials.Config.MobileKey)) + // Reconciling with the same set again changes nothing. + env.ReconcileCredentials(set) creds = env.GetCredentials() assert.Len(t, creds, 2) assert.Contains(t, creds, envConfig.SDKKey) - assert.Contains(t, creds, st.EnvWithAllCredentials.Config.MobileKey) + assert.Contains(t, creds, mobileKey) } func TestChangeSDKKey(t *testing.T) { @@ -246,6 +266,7 @@ func TestChangeSDKKey(t *testing.T) { env := makeBasicEnv(t, envConfig, clientFactory, mockLog.Loggers, readyCh) defer env.Close() + envImpl := env.(*envContextImpl) assert.Equal(t, env, requireEnvReady(t, readyCh)) client1 := requireClientReady(t, clientCh) @@ -257,14 +278,16 @@ func TestChangeSDKKey(t *testing.T) { assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) assert.Empty(t, env.GetDeprecatedCredentials()) - // For the purposes of key rotation, we'll make time deterministic. + // For the purposes of key rotation, we'll make time deterministic. We drive the grace-period + // setup through the legacy UpdateCredential API (with an injected time), then advance the cleanup + // ticker (triggerCredentialChanges) to expire the deprecated key — the same path the periodic + // ticker uses in production. start := time.Unix(1000, 0) - // Upon rotating to key2, the original key should still be valid for a hour. - env.UpdateCredential( - NewCredentialUpdate(key2). - WithTime(start). - WithGracePeriod(envConfig.SDKKey, start.Add(1*time.Hour))) + // Upon rotating to key2, the original key should still be valid for an hour. + envImpl.UpdateCredential(NewCredentialUpdate(key2). + WithGracePeriod(envConfig.SDKKey, start.Add(1 * time.Hour)). + WithTime(start)) assert.Equal(t, []credential.SDKCredential{key2}, env.GetCredentials()) assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetDeprecatedCredentials()) @@ -283,14 +306,14 @@ func TestChangeSDKKey(t *testing.T) { } // Simulate an amount of time passing that is less than the deprecation period. The original key should still be valid. - env.UpdateCredential(NewCredentialUpdate(key2).WithTime(start.Add(45 * time.Minute))) + envImpl.triggerCredentialChanges(start.Add(45 * time.Minute)) if !helpers.AssertChannelNotClosed(t, client1.CloseCh, 1*time.Second, "client for envConfig.SDKKey should not have been closed yet") { t.FailNow() } // We are now an instant after the deprecation period. This should cause the original key to become expired // and trigger the client to close. - env.UpdateCredential(NewCredentialUpdate(key2).WithTime(start.Add(1*time.Hour + 1*time.Millisecond))) + envImpl.triggerCredentialChanges(start.Add(1*time.Hour + 1*time.Millisecond)) assert.Equal(t, []credential.SDKCredential{key2}, env.GetCredentials()) assert.Empty(t, env.GetDeprecatedCredentials()) @@ -300,6 +323,105 @@ func TestChangeSDKKey(t *testing.T) { } +func TestNonAnchorSDKKeysDoNotOpenUpstreamClient(t *testing.T) { + envConfig := st.EnvMain.Config + readyCh := make(chan EnvContext, 1) + // Buffer large enough to catch any unexpected extra clients. + clientCh := make(chan *testclient.FakeLDClient, 10) + clientFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + env := makeBasicEnv(t, envConfig, clientFactory, mockLog.Loggers, readyCh) + defer env.Close() + + // One client is opened for the anchor key during construction. + assert.Equal(t, env, requireEnvReady(t, readyCh)) + anchorClient := requireClientReady(t, clientCh) + assert.Equal(t, envConfig.SDKKey, anchorClient.Key) + + nonAnchorKey1 := config.SDKKey("non-anchor-key-1") + nonAnchorKey2 := config.SDKKey("non-anchor-key-2") + + // Reconcile to anchor + 2 non-anchor SDK keys. The anchor is unchanged, so no new anchor client + // is needed. Non-anchor keys must get envStreams + handlers + connection mapping but must NOT + // open an upstream client. + env.ReconcileCredentials( + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). + WithPrimarySDKKey(envConfig.SDKKey). + WithSDKKey(nonAnchorKey1). + WithSDKKey(nonAnchorKey2))) + + // All three SDK keys are accepted... + creds := env.GetCredentials() + assert.Contains(t, creds, envConfig.SDKKey) + assert.Contains(t, creds, nonAnchorKey1) + assert.Contains(t, creds, nonAnchorKey2) + + // ...but no additional upstream client was started. + if !helpers.AssertNoMoreValues(t, clientCh, 200*time.Millisecond) { + t.FailNow() + } +} + +func TestNonPrimaryMobileKeyDoesNotStealEventForwarding(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + envConfig := st.EnvWithAllCredentials.Config + primaryMobile := envConfig.MobileKey + nonPrimaryMobile := config.MobileKey("mob-non-primary") + + eventRecorderHandler, requestsCh := httphelpers.RecordingHandler(httphelpers.HandlerWithStatus(202)) + httphelpers.WithServer(eventRecorderHandler, func(server *httptest.Server) { + var allConfig config.Config + allConfig.Events.SendEvents = true + allConfig.Events.EventsURI, _ = configtypes.NewOptURLAbsoluteFromString(server.URL) + allConfig.Events.FlushInterval = configtypes.NewOptDuration(time.Millisecond * 10) + env, err := NewEnvContext(EnvContextImplParams{ + Identifiers: EnvIdentifiers{ConfiguredName: envName}, + EnvConfig: envConfig, + AllConfig: allConfig, + ClientFactory: testclient.FakeLDClientFactory(true), + Loggers: mockLog.Loggers, + ConnectionMapper: mockConnectionMapper{}, + }, nil) + require.NoError(t, err) + defer env.Close() + envImpl := env.(*envContextImpl) + + // Reconcile to a set that keeps the original mobile key as primary but also accepts a second, + // non-primary mobile key. Accepting the non-primary key must NOT repoint event forwarding — + // events collapse to the primary mobile key, mirroring the SDK anchor. + env.ReconcileCredentials(mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). + WithPrimarySDKKey(envConfig.SDKKey). + WithPrimaryMobileKey(primaryMobile). + WithMobileKey(nonPrimaryMobile). + WithEnvironmentID(envConfig.EnvID))) + + ed := envImpl.GetEventDispatcher() + require.NotNil(t, ed) + handler := ed.GetHandler(basictypes.MobileSDK, ldevents.AnalyticsEventDataKind) + require.NotNil(t, handler) + + rr := httptest.NewRecorder() + headers := make(http.Header) + headers.Set("Content-Type", "application/json") + headers.Set("Authorization", string(primaryMobile)) + headers.Set("X-LaunchDarkly-Event-Schema", strconv.Itoa(events.SummaryEventsSchemaVersion)) + body := `[{"kind":"identify","creationDate":1000,"key":"userkey","user":{"key":"userkey"}}]` + req := st.BuildRequest("POST", server.URL+"/mobile/events/bulk", []byte(body), headers) + handler(rr, req) + require.Equal(t, 202, rr.Result().StatusCode) + + // Mobile events forward under the env's primary mobile key, not the freshly-accepted + // non-primary one. + eventPost := helpers.RequireValue(t, requestsCh, time.Second) + assert.Equal(t, string(primaryMobile), eventPost.Request.Header.Get("Authorization")) + }) +} + func TestSDKClientCreationFails(t *testing.T) { envConfig := st.EnvWithAllCredentials.Config envConfig.TTL = configtypes.NewOptDuration(time.Hour) diff --git a/internal/relayenv/env_context_reanchor_test.go b/internal/relayenv/env_context_reanchor_test.go index e5436147..e6078a05 100644 --- a/internal/relayenv/env_context_reanchor_test.go +++ b/internal/relayenv/env_context_reanchor_test.go @@ -11,9 +11,9 @@ package relayenv // .agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md. // // Terminology: "re-anchor" = swapping the single upstream SDK client when sdkKey.value changes. -// Today there is no dedicated re-anchor method; the closest existing path is UpdateCredential with a -// grace period (which rotates the primary SDK key and stands up a new client), so several tests drive -// that path and observe where it falls short of the §7 requirements. +// Today there is no dedicated re-anchor method; the closest existing path is ReconcileCredentials with +// an expiring (grace-period) key (which rotates the primary SDK key and stands up a new client), so +// several tests drive that path and observe where it falls short of the §7 requirements. import ( "errors" @@ -95,12 +95,15 @@ func (f *sharedStoreFactory) Build(_ subsystems.ClientContext) (subsystems.DataS return f.store, nil } -// newReanchorCredentialUpdate builds a CredentialUpdate that rotates the primary SDK key to newKey -// while keeping oldKey valid for a grace hour (so the old client is not torn down during the swap). -// This mirrors the backend's default-rotation behavior: the new anchor is non-expiring, the demoted -// old anchor carries an expiry. -func newReanchorCredentialUpdate(newKey, oldKey config.SDKKey, now time.Time) *CredentialUpdate { - return NewCredentialUpdate(newKey).WithTime(now).WithGracePeriod(oldKey, now.Add(time.Hour)) +// reanchor re-anchors env onto newKey while keeping oldKey valid for a grace hour (so the old client +// is not torn down during the swap). This mirrors the backend's default-rotation behavior: the new +// anchor is non-expiring, the demoted old anchor carries an expiry. It drives the time-injectable +// reconcileCredentials directly so the grace-period math is deterministic. +func reanchor(t *testing.T, env EnvContext, newKey, oldKey config.SDKKey, now time.Time) { + t.Helper() + env.(*envContextImpl).UpdateCredential(NewCredentialUpdate(newKey). + WithGracePeriod(oldKey, now.Add(time.Hour)). + WithTime(now)) } // ----------------------------------------------------------------------------------------------- @@ -227,7 +230,7 @@ func TestReanchorPoC_H2_DownstreamConnectionSurvivesReAnchor(t *testing.T) { // The connection is keyed on the env ID (a ScopedCredential), independent of the upstream SDK // key, so swapping the SDK anchor must not disturb it. start := time.Unix(1000, 0) - env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) // The new anchor client comes up on a background goroutine. Credential additions start the client // with a nil readyCh (so it does NOT signal sdkStartedCh); wait on the credential set instead. @@ -340,7 +343,7 @@ func TestReanchorPoC_H3_BigSegmentSyncIsNotReWiredOnReAnchor(t *testing.T) { // Re-anchor onto a new SDK key. start := time.Unix(1000, 0) - env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) require.Eventually(t, func() bool { for _, c := range env.GetCredentials() { if c == reanchorTestKey2 { @@ -436,7 +439,7 @@ func TestReanchorPoC_H5_InMemoryStoreIsWipedByReAnchor(t *testing.T) { // Re-anchor onto a new key (old key kept valid for a grace hour, so the old client is not closed -- // i.e. this exercises the recommended "start-new-before-close-old" ordering). start := time.Unix(1000, 0) - env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) client2 := requireClientReady(t, clientCh) require.Eventually(t, func() bool { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond, @@ -504,7 +507,7 @@ func TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow(t *testing.T) { // Re-anchor onto a new key. start := time.Unix(1000, 0) - env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) client2 := requireClientReady(t, clientCh) require.Eventually(t, func() bool { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond) @@ -549,10 +552,10 @@ func TestReanchorPoC_H6_AnchorPointerFlipsBeforeNewClientIsRegistered(t *testing client1 := requireClientReady(t, clientCh) require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) - // Re-anchor. UpdateCredential flips the rotator's primary SDK key synchronously, then starts the new - // client on a background goroutine (which blocks in the factory on `gate`). + // Re-anchor. reconcileCredentials flips the rotator's primary SDK key synchronously, then starts the + // new client on a background goroutine (which blocks in the factory on `gate`). start := time.Unix(1000, 0) - env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) // FINDING: there is a window where the anchor pointer already names the new key but no client exists // for it yet, so GetClient() returns nil. GetClient() == clients[rotator.SDKKey()], and the rotator's @@ -600,7 +603,7 @@ func TestReanchorPoC_H7_FailedNewClientLeavesEnvWithoutAnchorClient(t *testing.T // Re-anchor onto a key whose client init fails (old key kept valid for a grace hour). start := time.Unix(1000, 0) - env.UpdateCredential(newReanchorCredentialUpdate(reanchorTestKey2, envConfig.SDKKey, start)) + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) require.Eventually(t, func() bool { return env.GetInitError() != nil }, time.Second, 10*time.Millisecond, "the failed new-client init should surface as an init error") diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index 9588e8cf..57625701 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -31,6 +31,7 @@ func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentPar env, _, err := a.r.addEnvironment(params.Identifiers, envConfig, nil) if err != nil { a.r.loggers.Errorf(logMsgAutoConfEnvInitError, params.Identifiers.GetDisplayName(), err) + return } if params.ExpiringSDKKey.Defined() { diff --git a/relay/autoconfig_actions_test.go b/relay/autoconfig_actions_test.go index c957cf1a..2ae5ce16 100644 --- a/relay/autoconfig_actions_test.go +++ b/relay/autoconfig_actions_test.go @@ -136,13 +136,11 @@ func TestAutoConfigInitWithExpiringSDKKey(t *testing.T) { } initialEvent := makeAutoConfPutEvent(envWithKeys) autoConfTest(t, testAutoConfDefaultConfig, &initialEvent, func(p autoConfTestParams) { - client1 := p.awaitClient() - client2 := p.awaitClient() - if client1.Key == oldKey { - client1, client2 = client2, client1 - } - assert.Equal(t, newKey, client1.Key) - assert.Equal(t, oldKey, client2.Key) + // Only the anchor (newKey) opens an upstream client; the expiring oldKey is accepted + // locally but shares the anchor's connection (anchor-only upstream client). + anchorClient := p.awaitClient() + assert.Equal(t, newKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) env := p.awaitEnvironment(envWithKeys.id) assertEnvProps(t, envWithKeys.params(), env) @@ -220,13 +218,11 @@ func TestAutoConfigAddEnvironmentWithExpiringSDKKey(t *testing.T) { autoConfTest(t, testAutoConfDefaultConfig, &initialEvent, func(p autoConfTestParams) { p.stream.Enqueue(makeAutoConfPatchEvent(envWithKeys)) - client1 := p.awaitClient() - client2 := p.awaitClient() - if client1.Key == oldKey { - client1, client2 = client2, client1 - } - assert.Equal(t, newKey, client1.Key) - assert.Equal(t, oldKey, client2.Key) + // Only the anchor (newKey) opens an upstream client; the expiring oldKey is accepted + // locally but shares the anchor's connection (anchor-only upstream client). + anchorClient := p.awaitClient() + assert.Equal(t, newKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) env := p.awaitEnvironment(envWithKeys.id) assertEnvProps(t, envWithKeys.params(), env) @@ -237,10 +233,38 @@ func TestAutoConfigAddEnvironmentWithExpiringSDKKey(t *testing.T) { paramsWithOldKey := envWithKeys.params() paramsWithOldKey.SDKKey = oldKey p.assertEnvLookup(env, paramsWithOldKey) + }) +} - if !helpers.AssertChannelNotClosed(t, client2.CloseCh, time.Millisecond*300, "should not have closed client for deprecated key yet") { - t.FailNow() - } +// When addEnvironment fails, the auto-config handler must not go on to call UpdateCredential on +// the nil EnvContext it got back. This is only reachable when the payload also carries an expiring +// SDK key (the gate that triggers the credential update). We force the failure deterministically by +// closing the Relay first, so addEnvironment returns errAlreadyClosed with a nil env. +func TestAutoConfigAddEnvironmentWithExpiringSDKKeyDoesNotPanicWhenInitFails(t *testing.T) { + newKey := c.SDKKey("newsdkkey") + oldKey := c.SDKKey("oldsdkkey") + envWithKeys := testAutoConfEnv1 + envWithKeys.sdkKey = envfactory.SDKKeyRep{ + Value: newKey, + Expiring: envfactory.ExpiringKeyRep{ + Value: oldKey, + Timestamp: ldtime.UnixMillisNow() + 100000, + }, + } + + initialEvent := makeAutoConfPutEvent() + autoConfTest(t, testAutoConfDefaultConfig, &initialEvent, func(p autoConfTestParams) { + params := envWithKeys.params() + require.True(t, params.ExpiringSDKKey.Defined(), + "precondition: params must carry an expiring SDK key to reach the credential-update branch") + + // Closing the Relay makes the next addEnvironment return (nil, nil, errAlreadyClosed). + require.NoError(t, p.relay.Close()) + + actions := &relayAutoConfigActions{r: p.relay} + require.NotPanics(t, func() { + actions.AddEnvironment(params) + }, "AddEnvironment must not dereference a nil EnvContext when addEnvironment fails") }) } From d824d5da9240fd1f789a39fe08f51e9fac8fb7dc Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 24 Jun 2026 12:28:10 -0700 Subject: [PATCH 17/66] feat(envfactory): shared BuildAcceptedSet helper for ReconcileCredentials (#713) Introduces BuildAcceptedSet, a shared envfactory helper that turns EnvironmentParams into a credential.AcceptedSet (plus anchor SDK key) for upcoming EnvContext.ReconcileCredentials use in autoconfig and filedata handlers. Handler wiring is intentionally deferred. --- internal/envfactory/reconcile_helper.go | 65 ++++ internal/envfactory/reconcile_helper_test.go | 352 +++++++++++++++++++ 2 files changed, 417 insertions(+) create mode 100644 internal/envfactory/reconcile_helper.go create mode 100644 internal/envfactory/reconcile_helper_test.go diff --git a/internal/envfactory/reconcile_helper.go b/internal/envfactory/reconcile_helper.go new file mode 100644 index 00000000..db9a8781 --- /dev/null +++ b/internal/envfactory/reconcile_helper.go @@ -0,0 +1,65 @@ +package envfactory + +import ( + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/credential" +) + +// BuildAcceptedSet converts an EnvironmentParams into the AcceptedSet and anchor +// credential needed by EnvContext.ReconcileCredentials. +// +// Credential identity is keyed by value (the secret string), not by key (the human-readable +// identifier). A rename — same value, different identifier — therefore produces the same +// AcceptedSet as if nothing changed. +// +// Expiry comes from AcceptedSDKKey.Expiry / AcceptedMobileKey.Expiry (the arrays). The legacy +// sdkKey.expiring wire slot is never consulted here — relay trusts the array. +// +// The anchor (params.SDKKey) is added and designated as the primary SDK key, and the primary +// mobile key (params.MobileKey) is added and designated, in addition to the full accepted arrays. +// The builder de-duplicates by value, so an anchor or primary mobile key that also appears in its +// array is added only once. +// +// If no anchor is designated — params.SDKKey is undefined — no set is built and a +// *credential.MalformedCredentialSetError is returned with an empty AcceptedSet. The caller must +// preserve the previous accepted state and, for RAC handlers, reconnect the stream with jitter to +// force a fresh put. Structural validation of the wire payload (undefined credentials, an anchor +// absent from the array) happens upstream when the payload is parsed into params. +func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config.SDKKey, error) { + anchor := params.SDKKey + + // WithPrimarySDKKey / WithPrimaryMobileKey each add the key and designate it (the anchor and the + // wire's mobKey, respectively). An undefined key makes the call a no-op, so an undefined anchor + // leaves the set with no designated anchor and Build returns a *MalformedCredentialSetError. + b := credential.NewAcceptedSetBuilder(). + WithEnvironmentID(params.EnvID). + WithPrimarySDKKey(anchor). + WithPrimaryMobileKey(params.MobileKey) + + // Add the remaining accepted keys. The builder de-duplicates by value, so the anchor and the + // primary mobile key — already added permanently above — are ignored when they reappear in + // their arrays. That also defends the anchor-never-expiring invariant: a payload that (wrongly) + // carries an expiry on the anchor's own entry cannot demote it, because the permanent anchor is + // already present. + for _, k := range params.AcceptedSDKKeys { + if k.Expiry.IsZero() { + b.WithSDKKey(k.Value) + } else { + b.WithExpiringSDKKey(k.Value, k.Expiry) + } + } + + for _, k := range params.AcceptedMobileKeys { + if k.Expiry.IsZero() { + b.WithMobileKey(k.Value) + } else { + b.WithExpiringMobileKey(k.Value, k.Expiry) + } + } + + set, err := b.Build() + if err != nil { + return credential.AcceptedSet{}, anchor, err + } + return set, anchor, nil +} diff --git a/internal/envfactory/reconcile_helper_test.go b/internal/envfactory/reconcile_helper_test.go new file mode 100644 index 00000000..2c244711 --- /dev/null +++ b/internal/envfactory/reconcile_helper_test.go @@ -0,0 +1,352 @@ +package envfactory + +import ( + "errors" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/credential" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// expiry1 is a fixed future timestamp used in tests to represent an expiring key. +var expiry1 = time.Date(2099, 1, 1, 0, 0, 0, 0, time.UTC) + +// mustBuild builds the AcceptedSet from b, failing the test if Build returns an error. It is used to +// construct the expected set for comparison. +func mustBuild(t *testing.T, b *credential.AcceptedSetBuilder) credential.AcceptedSet { + t.Helper() + set, err := b.Build() + require.NoError(t, err) + return set +} + +// makeParams is a convenience builder for EnvironmentParams test fixtures. +// sdkKey is the anchor, sdkKeys are the full accepted set (must include the anchor), +// mobileKey is the single (primary) mobile key to include. +func makeParams(sdkKey config.SDKKey, sdkKeys []AcceptedSDKKey, mobileKey config.MobileKey) EnvironmentParams { + mob := []AcceptedMobileKey{} + if mobileKey.Defined() { + mob = []AcceptedMobileKey{{Value: mobileKey}} + } + return EnvironmentParams{ + EnvID: config.EnvironmentID("env-abc"), + SDKKey: sdkKey, + MobileKey: mobileKey, + AcceptedSDKKeys: sdkKeys, + AcceptedMobileKeys: mob, + } +} + +// TestBuildAcceptedSet_HappyPath verifies the basic case: a single permanent SDK key that is the +// anchor, plus a mobile key and env ID. +func TestBuildAcceptedSet_HappyPath(t *testing.T) { + params := makeParams( + "sdk-anchor", + []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, + "mob-primary", + ) + set, anchor, err := BuildAcceptedSet(params) + + require.NoError(t, err) + assert.Equal(t, config.SDKKey("sdk-anchor"), anchor) + + expected := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithPrimarySDKKey("sdk-anchor"). + WithPrimaryMobileKey("mob-primary")) + assert.Equal(t, expected, set) +} + +// TestBuildAcceptedSet_MultipleKeys verifies that multiple accepted SDK keys (anchor + non-anchor +// permanent + expiring non-anchor) are all included in the returned AcceptedSet. +func TestBuildAcceptedSet_MultipleKeys(t *testing.T) { + params := makeParams( + "sdk-anchor", + []AcceptedSDKKey{ + {Key: "default", Value: "sdk-anchor"}, + {Key: "service-a", Value: "sdk-service-a"}, // permanent, non-anchor + {Key: "old-key", Value: "sdk-old", Expiry: expiry1}, // expiring, non-anchor + }, + "mob-primary", + ) + set, anchor, err := BuildAcceptedSet(params) + + require.NoError(t, err) + assert.Equal(t, config.SDKKey("sdk-anchor"), anchor) + + expected := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithPrimarySDKKey("sdk-anchor"). + WithSDKKey("sdk-service-a"). + WithExpiringSDKKey("sdk-old", expiry1). + WithPrimaryMobileKey("mob-primary")) + assert.Equal(t, expected, set) +} + +// TestBuildAcceptedSet_Rename verifies that a rename — same credential value, different key +// identifier — is a no-op: the returned AcceptedSet is identical regardless of the identifier. +func TestBuildAcceptedSet_Rename(t *testing.T) { + // Build AcceptedSet for the "before" and "after" of a rename. + paramsOldName := makeParams( + "sdk-anchor", + []AcceptedSDKKey{{Key: "old-name", Value: "sdk-anchor"}}, + "mob-primary", + ) + paramsNewName := makeParams( + "sdk-anchor", + []AcceptedSDKKey{{Key: "new-name", Value: "sdk-anchor"}}, + "mob-primary", + ) + + setOld, _, errOld := BuildAcceptedSet(paramsOldName) + setNew, _, errNew := BuildAcceptedSet(paramsNewName) + + require.NoError(t, errOld) + require.NoError(t, errNew) + assert.Equal(t, setOld, setNew, "rename (same value, different key identifier) should produce the same AcceptedSet") +} + +// TestBuildAcceptedSet_Deexpiry verifies that removing the expiry from an existing key (a +// previously expiring key that is now permanent) results in the key being permanent in the +// returned AcceptedSet. The "cancel scheduled drop" effect is realized when ReconcileCredentials +// applies this set to the Rotator. +func TestBuildAcceptedSet_Deexpiry(t *testing.T) { + // "Before" state: sdk-old has an expiry. + paramsWithExpiry := makeParams( + "sdk-anchor", + []AcceptedSDKKey{ + {Key: "default", Value: "sdk-anchor"}, + {Key: "old-key", Value: "sdk-old", Expiry: expiry1}, + }, + "mob-primary", + ) + // "After" state: sdk-old's expiry is removed — it is now permanent. + paramsNoExpiry := makeParams( + "sdk-anchor", + []AcceptedSDKKey{ + {Key: "default", Value: "sdk-anchor"}, + {Key: "old-key", Value: "sdk-old"}, // Expiry zero = permanent + }, + "mob-primary", + ) + + setWithExpiry, _, errWithExpiry := BuildAcceptedSet(paramsWithExpiry) + setNoExpiry, _, errNoExpiry := BuildAcceptedSet(paramsNoExpiry) + + require.NoError(t, errWithExpiry) + require.NoError(t, errNoExpiry) + + // The set built without expiry must include sdk-old as a permanent key. + expectedPermanent := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithPrimarySDKKey("sdk-anchor"). + WithSDKKey("sdk-old"). // permanent, no expiry + WithPrimaryMobileKey("mob-primary")) + assert.Equal(t, expectedPermanent, setNoExpiry) + + // Sanity: the expiring and non-expiring versions are different. + assert.NotEqual(t, setWithExpiry, setNoExpiry) +} + +// TestBuildAcceptedSet_AnchorNotInArray verifies that an anchor absent from AcceptedSDKKeys is no +// longer rejected here: WithPrimarySDKKey adds and designates the anchor regardless, so the +// resulting set contains both the anchor and the array entry. Structural validation of the wire +// payload (anchor-absent-from-array) happens upstream when the payload is parsed into params. +func TestBuildAcceptedSet_AnchorNotInArray(t *testing.T) { + params := makeParams( + "sdk-anchor", + []AcceptedSDKKey{ + {Key: "other-key", Value: "sdk-other"}, // anchor NOT in the array + }, + "mob-primary", + ) + set, anchor, err := BuildAcceptedSet(params) + + require.NoError(t, err) + assert.Equal(t, config.SDKKey("sdk-anchor"), anchor) + + expected := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithPrimarySDKKey("sdk-anchor"). // added + designated even though absent from the array + WithSDKKey("sdk-other"). + WithPrimaryMobileKey("mob-primary")) + assert.Equal(t, expected, set) +} + +// TestBuildAcceptedSet_AnchorUndefined verifies that an undefined anchor (empty SDKKey) yields a +// *credential.MalformedCredentialSetError: no anchor was designated, so Build rejects the set. +func TestBuildAcceptedSet_AnchorUndefined(t *testing.T) { + params := makeParams( + "", // undefined anchor + []AcceptedSDKKey{ + {Key: "key-a", Value: "sdk-a"}, + }, + "mob-primary", + ) + _, _, err := BuildAcceptedSet(params) + + require.Error(t, err) + var malformed *credential.MalformedCredentialSetError + require.True(t, errors.As(err, &malformed)) + // An undefined anchor must produce the "missing" message, not the "not present" one. This + // only holds if Anchor is an untyped nil — a boxed zero-value config.SDKKey would be non-nil + // and route Error() down the wrong branch. + assert.Nil(t, malformed.Anchor) + assert.Contains(t, malformed.Error(), "anchor SDK key is missing") +} + +// TestBuildAcceptedSet_NoSDKKeys verifies that when neither an anchor nor any array SDK keys are +// present, Build returns an error (the set has no SDK key at all). +func TestBuildAcceptedSet_NoSDKKeys(t *testing.T) { + params := EnvironmentParams{ + SDKKey: "", // undefined anchor + AcceptedSDKKeys: []AcceptedSDKKey{}, + AcceptedMobileKeys: []AcceptedMobileKey{}, + } + _, _, err := BuildAcceptedSet(params) + require.Error(t, err, "a set with no SDK key at all must be rejected") +} + +// TestBuildAcceptedSet_MixedUpdate verifies add + re-anchor + remove in a single params update +// produces an AcceptedSet that contains the right keys in the right state. The ordering +// (add → re-anchor → remove) is enforced by ReconcileCredentials when it consumes this set; +// this test only asserts the AcceptedSet content. +func TestBuildAcceptedSet_MixedUpdate(t *testing.T) { + // New state after the patch: + // - sdk-new-anchor is the new anchor (re-anchor) + // - sdk-b carries over unchanged + // - sdk-c is newly added + // - sdk-old-anchor is gone (remove) + params := makeParams( + "sdk-new-anchor", + []AcceptedSDKKey{ + {Key: "new-default", Value: "sdk-new-anchor"}, // re-anchor + {Key: "service-b", Value: "sdk-b"}, // unchanged + {Key: "service-c", Value: "sdk-c"}, // added + }, + "mob-primary", + ) + set, anchor, err := BuildAcceptedSet(params) + + require.NoError(t, err) + assert.Equal(t, config.SDKKey("sdk-new-anchor"), anchor) + + expected := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithPrimarySDKKey("sdk-new-anchor"). + WithSDKKey("sdk-b"). + WithSDKKey("sdk-c"). + WithPrimaryMobileKey("mob-primary")) + assert.Equal(t, expected, set) +} + +// TestBuildAcceptedSet_AnchorNeverExpiring verifies the invariant defense: even if a payload +// carries an expiry on the anchor's own entry, the anchor is added as a permanent key, never +// as an expiring one. +func TestBuildAcceptedSet_AnchorNeverExpiring(t *testing.T) { + params := makeParams( + "sdk-anchor", + []AcceptedSDKKey{ + {Key: "default", Value: "sdk-anchor", Expiry: expiry1}, // anchor with a bogus expiry + {Key: "service-a", Value: "sdk-service-a"}, + }, + "mob-primary", + ) + set, anchor, err := BuildAcceptedSet(params) + + require.NoError(t, err) + assert.Equal(t, config.SDKKey("sdk-anchor"), anchor) + + // Anchor is permanent (WithPrimarySDKKey), not expiring — identical to a payload with no anchor expiry. + expected := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithPrimarySDKKey("sdk-anchor"). + WithSDKKey("sdk-service-a"). + WithPrimaryMobileKey("mob-primary")) + assert.Equal(t, expected, set) +} + +// TestBuildAcceptedSet_MultipleMobileKeys verifies that all accepted mobile keys are included, +// exercising the len(AcceptedMobileKeys) > 1 path, and that the wire's mobKey is designated as the +// primary mobile key. +func TestBuildAcceptedSet_MultipleMobileKeys(t *testing.T) { + params := EnvironmentParams{ + EnvID: "env-abc", + SDKKey: "sdk-anchor", + MobileKey: "mob-primary", + AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, + AcceptedMobileKeys: []AcceptedMobileKey{ + {Key: "mob-1", Value: "mob-primary"}, + {Key: "mob-2", Value: "mob-secondary"}, + }, + } + set, _, err := BuildAcceptedSet(params) + + require.NoError(t, err) + expected := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithPrimarySDKKey("sdk-anchor"). + WithMobileKey("mob-primary"). + WithMobileKey("mob-secondary"). + WithPrimaryMobileKey("mob-primary")) + assert.Equal(t, expected, set) +} + +// TestBuildAcceptedSet_ExpiringMobileKey verifies that a mobile key carrying a non-zero Expiry is +// plumbed through as an expiring key (parallel to the expiring-SDK-key path), while the permanent +// primary mobile key is designated. This is what makes per-key mobile expiry work end-to-end: +// params carry it → BuildAcceptedSet plumbs it into the AcceptedSet → Reconcile acts on it. +func TestBuildAcceptedSet_ExpiringMobileKey(t *testing.T) { + params := EnvironmentParams{ + EnvID: "env-abc", + SDKKey: "sdk-anchor", + MobileKey: "mob-primary", + AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, + AcceptedMobileKeys: []AcceptedMobileKey{ + {Key: "mob-1", Value: "mob-primary"}, // permanent primary + {Key: "mob-old", Value: "mob-old", Expiry: expiry1}, // expiring + }, + } + set, _, err := BuildAcceptedSet(params) + + require.NoError(t, err) + expected := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithPrimarySDKKey("sdk-anchor"). + WithMobileKey("mob-primary"). + WithExpiringMobileKey("mob-old", expiry1). + WithPrimaryMobileKey("mob-primary")) + assert.Equal(t, expected, set, "expiring mobile key must land as an expiring key in the set") +} + +// TestBuildAcceptedSet_TrustTheArray verifies that the legacy sdkKey.expiring slot is not +// consulted: when EnvironmentParams.ExpiringSDKKey is populated (from the legacy field) but +// AcceptedSDKKeys does NOT contain that key, the key is absent from the returned AcceptedSet. +func TestBuildAcceptedSet_TrustTheArray(t *testing.T) { + // Simulate an old-relay payload where ExpiringSDKKey is populated from sdkKey.expiring, + // but AcceptedSDKKeys only has the anchor (no expiring key in the array). + params := EnvironmentParams{ + EnvID: "env-abc", + SDKKey: "sdk-anchor", + MobileKey: "mob-primary", + ExpiringSDKKey: ExpiringSDKKey{ // legacy field — must NOT be consulted + Key: "sdk-legacy-expiring", + Expiration: expiry1, + }, + AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, + AcceptedMobileKeys: []AcceptedMobileKey{{Value: "mob-primary"}}, + } + + set, _, err := BuildAcceptedSet(params) + + require.NoError(t, err) + expected := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithPrimarySDKKey("sdk-anchor"). + WithPrimaryMobileKey("mob-primary")) + assert.Equal(t, expected, set, "legacy sdkKey.expiring slot must not appear in AcceptedSet") +} From 3b50d05ee880c8a6e24edd2767612157f98545a2 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 24 Jun 2026 13:39:35 -0700 Subject: [PATCH 18/66] feat(credential): generalize cleanup ticker for per-key expiry + mobile-key disconnects (#714) Generalizes Rotator.StepTime to enforce per-key expiry for mobile keys and for the reconcile path, symmetric with the existing SDK/legacy grace path. DeprecatedCredentials now surfaces deprecated mobile keys alongside deprecated SDK keys. --- internal/credential/rotator.go | 56 ++++++- internal/credential/rotator_test.go | 171 ++++++++++++++++++++- internal/relayenv/env_context_impl_test.go | 84 ++++++++++ 3 files changed, 300 insertions(+), 11 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 1305b120..62a95dc6 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -20,8 +20,8 @@ type Rotator struct { primaryMobileKey config.MobileKey // deprecatedMobileKeys stores mobile keys being phased out with a grace period, keyed - // by credential value with the associated expiry time. StepTime does not yet act on - // these entries; generalizing the cleanup ticker to drop them is handled separately. + // by credential value with the associated expiry time. StepTime walks this map and emits + // an expiration once a key's grace period passes, mirroring deprecatedSdkKeys. deprecatedMobileKeys map[config.MobileKey]time.Time // There is only one environment ID active at a given time, and it won't actually be rotated. The mechanism is @@ -141,10 +141,13 @@ func (r *Rotator) primaryCredentials() []SDKCredential { } func (r *Rotator) deprecatedCredentials() []SDKCredential { - deprecated := make([]SDKCredential, 0, len(r.deprecatedSdkKeys)) + deprecated := make([]SDKCredential, 0, len(r.deprecatedSdkKeys)+len(r.deprecatedMobileKeys)) for key := range r.deprecatedSdkKeys { deprecated = append(deprecated, key) } + for key := range r.deprecatedMobileKeys { + deprecated = append(deprecated, key) + } return deprecated } @@ -225,8 +228,8 @@ func (r *Rotator) updateEnvironmentID(envID config.EnvironmentID) { } // updateMobileKey sets a new primary mobile key. When grace is nil the outgoing key is -// immediately revoked; when non-nil its expiry is stored in deprecatedMobileKeys. -// StepTime does not yet act on deprecatedMobileKeys; that cleanup is handled separately. +// immediately revoked; when non-nil its expiry is stored in deprecatedMobileKeys for the +// cleanup ticker (StepTime) to act on. func (r *Rotator) updateMobileKey(mobileKey config.MobileKey, grace *GracePeriod) { r.mu.Lock() defer r.mu.Unlock() @@ -341,29 +344,66 @@ func (r *Rotator) expireSDKKey(sdkKey config.SDKKey) { r.expirations = append(r.expirations, sdkKey) } +// expireMobileKey drops a mobile key from both the deprecated grace map and the accepted set, then +// queues its expiration. Deleting from acceptedMobileKeys is load-bearing: PrimaryCredentials derives +// from that map, so an expired key would otherwise linger as a primary credential. Mirrors expireSDKKey. +func (r *Rotator) expireMobileKey(mobileKey config.MobileKey) { + r.loggers.Infof("Deprecated mobile key %s has expired and is no longer valid for authentication", mobileKey.Masked()) + delete(r.deprecatedMobileKeys, mobileKey) + delete(r.acceptedMobileKeys, mobileKey) + r.expirations = append(r.expirations, mobileKey) +} + // StepTime provides the current time to the Rotator, allowing it to compute the set of additions and expirations // for the tracked credentials since the last time this method was called. +// +// It enforces expiry from both expiry mechanisms, for both SDK and mobile keys: +// - The legacy grace-period maps (deprecatedSdkKeys / deprecatedMobileKeys), populated by the +// RotateWithGrace path, where the expiry lives in the map value. +// - The reconcile path, where per-key expiry is stored as data on the accepted entry +// (acceptedKeyInfo.expiry); a nil expiry means the key is permanent and is never expired here. +// +// Expiry is strict (now strictly after the expiry timestamp), consistent across all four loops. func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expirations []SDKCredential) { r.mu.Lock() defer r.mu.Unlock() + // Legacy grace-period deprecations (RotateWithGrace path). for key, expiry := range r.deprecatedSdkKeys { if now.After(expiry) { r.expireSDKKey(key) } } + for key, expiry := range r.deprecatedMobileKeys { + if now.After(expiry) { + r.expireMobileKey(key) + } + } + + // Reconcile-path per-key expiry, stored on the accepted entry. The anchor and primary mobile key + // carry a nil expiry, so this never drops them. + for key, info := range r.acceptedSDKKeys { + if info.expiry != nil && now.After(*info.expiry) { + r.expireSDKKey(key) + } + } + for key, info := range r.acceptedMobileKeys { + if info.expiry != nil && now.After(*info.expiry) { + r.expireMobileKey(key) + } + } additions, expirations = r.additions, r.expirations r.additions = nil r.expirations = nil - return + return additions, expirations } // Reconcile updates the rotator to match set. The set names its own anchor (the primary SDK key) and // primary mobile key. It diffs the desired accepted set against the current one and queues additions // and expirations (drained by the next StepTime call); keys newly present are accepted, and keys no -// longer present are revoked. Per-key expiry is stored as data on the accepted entry — grace-period -// deprecation and the cleanup ticker that drops expiring keys are handled separately. An undefined +// longer present are revoked. Per-key expiry is stored as data on the accepted entry; the cleanup +// ticker (StepTime) is what later acts on it, dropping a key once its expiry passes. An undefined // environment ID leaves the current one unchanged, since environments are removed via teardown // rather than reconcile. // diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 510e9cd5..88136eb1 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -287,10 +287,18 @@ func TestRotateWithGraceMobileKey(t *testing.T) { assert.Equal(t, mob2, rotator.MobileKey()) // mob2 is a new addition; mob1 is in the deprecated set (not yet expired), - // so it should not appear as an expiration here. Cleanup is deferred to T1.c. + // so it should not appear as an expiration here. additions, expirations := rotator.StepTime(start) assert.ElementsMatch(t, []SDKCredential{mob2}, additions) assert.Empty(t, expirations) + + // One moment past the grace period, the cleanup ticker expires mob1 and evicts it from the + // accepted set entirely. + additions, expirations = rotator.StepTime(expiry.Add(1 * time.Millisecond)) + assert.Empty(t, additions) + assert.ElementsMatch(t, []SDKCredential{mob1}, expirations) + assert.NotContains(t, rotator.PrimaryCredentials(), SDKCredential(mob1)) + assert.NotContains(t, rotator.DeprecatedCredentials(), SDKCredential(mob1)) }) t.Run("immediately revokes outgoing key when grace period is already expired", func(t *testing.T) { @@ -474,8 +482,9 @@ func TestReconcileRevokesOmittedKeys(t *testing.T) { } func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { - // The foundation stores per-key expiry but does not yet act on it (no grace-period deprecation, - // no cleanup ticker — those are handled separately). An expiring key is simply accepted. + // Reconcile stores per-key expiry as data on the accepted entry; before that expiry passes, an + // expiring key is simply accepted and non-deprecated. The cleanup ticker (StepTime) only acts on + // the expiry once it elapses — see TestReconcileExpiringKeysAreEvictedByStepTime. r := newTestRotator() anchor := config.SDKKey("anchor") expiringSDK := config.SDKKey("expiring-sdk") @@ -541,3 +550,159 @@ func TestReconcileClearsStaleDeprecationForAcceptedKey(t *testing.T) { assert.Contains(t, r.PrimaryCredentials(), SDKCredential(old)) assert.Empty(t, r.DeprecatedCredentials()) } + +func TestReconcileExpiringKeysAreEvictedByStepTime(t *testing.T) { + // End-to-end on the reconcile path: a reconcile records per-key expiry as data on the accepted + // entry, and the cleanup ticker (StepTime) later drops both the expiring SDK key and the expiring + // mobile key once their expiry elapses — without ever passing through the legacy deprecated maps. + // The anchor and primary mobile key carry no expiry and survive. + r := newTestRotator() + anchor := config.SDKKey("anchor") + expiringSDK := config.SDKKey("expiring-sdk") + mob := config.MobileKey("mob") + expiringMobile := config.MobileKey("expiring-mob") + now := time.Unix(1000, 0) + expiry := now.Add(time.Hour) + + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithPrimarySDKKey(anchor). + WithExpiringSDKKey(expiringSDK, expiry). + WithPrimaryMobileKey(mob). + WithExpiringMobileKey(expiringMobile, expiry)), + now) + additions, expirations := r.StepTime(now) + require.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, additions) + require.Empty(t, expirations) + + // At the exact expiry, expiry is strict (now must be strictly after), so nothing is dropped yet. + additions, expirations = r.StepTime(expiry) + assert.Empty(t, additions) + assert.Empty(t, expirations) + + // One moment past the expiry: both expiring keys are evicted; anchor and primary mobile survive. + additions, expirations = r.StepTime(expiry.Add(1 * time.Millisecond)) + assert.Empty(t, additions) + assert.ElementsMatch(t, []SDKCredential{expiringSDK, expiringMobile}, expirations) + assert.ElementsMatch(t, []SDKCredential{anchor, mob}, r.PrimaryCredentials()) + assert.NotContains(t, r.PrimaryCredentials(), SDKCredential(expiringSDK)) + assert.NotContains(t, r.PrimaryCredentials(), SDKCredential(expiringMobile)) +} + +func TestMobileKeyDeprecation(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + const ( + mob1 = config.MobileKey("mob1") + mob2 = config.MobileKey("mob2") + ) + + start := time.Unix(10000, 0) + halfTime := start.Add(30 * time.Second) + deprecationTime := start.Add(1 * time.Minute) + + rotator.Initialize([]SDKCredential{mob1}) + + rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), deprecationTime, halfTime)) + additions, expirations := rotator.StepTime(halfTime) + assert.ElementsMatch(t, []SDKCredential{mob2}, additions) + assert.Empty(t, expirations) + + // At the exact expiry, not yet expired. + additions, expirations = rotator.StepTime(deprecationTime) + assert.Empty(t, additions) + assert.Empty(t, expirations) + + // One moment past the expiry: mob1 is expired. + additions, expirations = rotator.StepTime(deprecationTime.Add(1 * time.Millisecond)) + assert.Empty(t, additions) + assert.ElementsMatch(t, []SDKCredential{mob1}, expirations) +} + +func TestManyConcurrentMobileKeyDeprecation(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + makeKey := func(i int) config.MobileKey { + return config.MobileKey(fmt.Sprintf("mob%v", i)) + } + + rotator.Initialize([]SDKCredential{makeKey(0)}) + + const numKeys = 50 + now := time.Unix(10000, 0) + expiryTime := now.Add(1 * time.Hour) + + var keysDeprecated []SDKCredential + var keysAdded []SDKCredential + + for i := 0; i < numKeys; i++ { + nextKey := makeKey(i + 1) + keysDeprecated = append(keysDeprecated, makeKey(i)) + keysAdded = append(keysAdded, nextKey) + rotator.RotateWithGrace(nextKey, NewGracePeriod(config.SDKKey(""), expiryTime, now)) + } + + assert.Equal(t, keysAdded[len(keysAdded)-1], rotator.MobileKey()) + + // Until and including the exact expiry timestamp, no expirations. + additions, expirations := rotator.StepTime(expiryTime) + assert.ElementsMatch(t, keysAdded, additions) + assert.Empty(t, expirations) + + // One moment after the expiry time: batch of expirations. + additions, expirations = rotator.StepTime(expiryTime.Add(1 * time.Millisecond)) + assert.Empty(t, additions) + assert.ElementsMatch(t, keysDeprecated, expirations) +} + +func TestMixedSDKAndMobileKeyExpiry(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + sdk1 := config.SDKKey("sdk1") + sdk2 := config.SDKKey("sdk2") + mob1 := config.MobileKey("mob1") + mob2 := config.MobileKey("mob2") + + now := time.Unix(10000, 0) + expiry := now.Add(1 * time.Hour) + + rotator.Initialize([]SDKCredential{sdk1, mob1}) + + rotator.RotateWithGrace(sdk2, NewGracePeriod(sdk1, expiry, now)) + rotator.StepTime(now) + + rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), expiry, now)) + rotator.StepTime(now) + + // Both sdk1 and mob1 should expire at the same tick. + additions, expirations := rotator.StepTime(expiry.Add(1 * time.Millisecond)) + assert.Empty(t, additions) + assert.ElementsMatch(t, []SDKCredential{sdk1, mob1}, expirations) +} + +func TestDeprecatedCredentialsIncludesMobileKeys(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + sdk1 := config.SDKKey("sdk1") + sdk2 := config.SDKKey("sdk2") + mob1 := config.MobileKey("mob1") + mob2 := config.MobileKey("mob2") + + now := time.Unix(10000, 0) + expiry := now.Add(1 * time.Hour) + + rotator.Initialize([]SDKCredential{sdk1, mob1}) + + rotator.RotateWithGrace(sdk2, NewGracePeriod(sdk1, expiry, now)) + rotator.StepTime(now) + + rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), expiry, now)) + rotator.StepTime(now) + + deprecated := rotator.DeprecatedCredentials() + assert.ElementsMatch(t, []SDKCredential{sdk1, mob1}, deprecated) +} diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 8915e581..38a2ebde 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -323,6 +323,90 @@ func TestChangeSDKKey(t *testing.T) { } +// TestMobileKeyGraceExpiry drives a deprecated mobile key (legacy RotateWithGrace path) to expiry +// through the cleanup ticker (triggerCredentialChanges → StepTime), mirroring the SDK-key flow in +// TestChangeSDKKey. +func TestMobileKeyGraceExpiry(t *testing.T) { + envConfig := st.EnvMobile.Config + readyCh := make(chan EnvContext, 1) + + mob1 := envConfig.MobileKey + mob2 := config.MobileKey("mob2-new-key") + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactory(true), mockLog.Loggers, readyCh) + defer env.Close() + envImpl := env.(*envContextImpl) + + assert.Equal(t, env, requireEnvReady(t, readyCh)) + + start := time.Unix(2000, 0) + graceDuration := 1 * time.Hour + + // Rotate mob1 → mob2 with a grace period; mob1 is deprecated but still valid. + envImpl.keyRotator.RotateWithGrace(mob2, credential.NewGracePeriod(config.SDKKey(""), start.Add(graceDuration), start)) + envImpl.triggerCredentialChanges(start) + + assert.Contains(t, env.GetDeprecatedCredentials(), mob1) + + // Before the grace period ends, mob1 is still deprecated (not yet expired). + envImpl.triggerCredentialChanges(start.Add(30 * time.Minute)) + assert.Contains(t, env.GetDeprecatedCredentials(), mob1) + + // One moment past the grace period: the cleanup ticker evicts mob1 entirely. + envImpl.triggerCredentialChanges(start.Add(graceDuration + 1*time.Millisecond)) + + assert.NotContains(t, env.GetCredentials(), mob1) + assert.NotContains(t, env.GetDeprecatedCredentials(), mob1) +} + +// TestMobileKeyReconcileExpiry drives a mobile key carrying a per-key expiry end-to-end through the +// reconcile path: ReconcileCredentials records the expiry as data on the accepted entry, and the +// cleanup ticker (triggerCredentialChanges → StepTime) later evicts the key once its expiry elapses. +func TestMobileKeyReconcileExpiry(t *testing.T) { + envConfig := st.EnvMobile.Config + readyCh := make(chan EnvContext, 1) + + primaryMobile := envConfig.MobileKey + expiringMobile := config.MobileKey("mob-expiring") + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactory(true), mockLog.Loggers, readyCh) + defer env.Close() + envImpl := env.(*envContextImpl) + + assert.Equal(t, env, requireEnvReady(t, readyCh)) + + start := time.Unix(2000, 0) + expiry := start.Add(1 * time.Hour) + + // Reconcile to a set that accepts the primary mobile key (permanent) plus a second mobile key that + // carries a per-key expiry. + envImpl.reconcileCredentials( + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). + WithPrimarySDKKey(envConfig.SDKKey). + WithPrimaryMobileKey(primaryMobile). + WithExpiringMobileKey(expiringMobile, expiry)), + start) + + // Reconcile stores the expiry as data, so before it elapses the key is accepted (not deprecated). + assert.Contains(t, env.GetCredentials(), expiringMobile) + assert.NotContains(t, env.GetDeprecatedCredentials(), expiringMobile) + + // Halfway through, still accepted. + envImpl.triggerCredentialChanges(start.Add(30 * time.Minute)) + assert.Contains(t, env.GetCredentials(), expiringMobile) + + // One moment past expiry: the cleanup ticker evicts the expiring mobile key; the primary survives. + envImpl.triggerCredentialChanges(expiry.Add(1 * time.Millisecond)) + assert.NotContains(t, env.GetCredentials(), expiringMobile) + assert.Contains(t, env.GetCredentials(), primaryMobile) +} + func TestNonAnchorSDKKeysDoNotOpenUpstreamClient(t *testing.T) { envConfig := st.EnvMain.Config readyCh := make(chan EnvContext, 1) From 2f36d6a77de6a82ae9c010cc940b199544096fd1 Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Wed, 24 Jun 2026 19:03:13 -0400 Subject: [PATCH 19/66] fix: Close orphaned and stale SDK clients during key rotation (#716) Fixes SDK key rotation leaks and incorrect credential lifetime behavior in the Relay Proxy. --- internal/credential/rotator_test.go | 59 +++++++ internal/relayenv/env_context_impl.go | 42 ++++- internal/relayenv/env_context_impl_test.go | 169 +++++++++++++++++++++ 3 files changed, 268 insertions(+), 2 deletions(-) diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 88136eb1..3e0c4acc 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -237,6 +237,65 @@ func TestSDKKeyExpiredInThePastIsNotAdded(t *testing.T) { assert.Empty(t, expirations) } +func TestSDKKeyDeprecationWithAlreadyExpiredGraceRevokesPreviousPrimary(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + key1 := config.SDKKey("key1") + key2 := config.SDKKey("key2") + + expiry := time.Unix(10000, 0) + now := expiry.Add(1 * time.Hour) // now is after the grace period's expiry + + rotator.Initialize([]SDKCredential{key1}) + + // Rotate key1 -> key2, but the deprecation grace for the outgoing key1 has already elapsed. + // key2 becomes primary; key1 must be revoked immediately rather than lingering forever as an + // accepted-but-untracked key. (This mirrors the equivalent mobile-key behavior.) + rotator.RotateWithGrace(key2, NewGracePeriod(key1, expiry, now)) + + assert.Equal(t, key2, rotator.SDKKey()) + + additions, expirations := rotator.StepTime(now) + assert.ElementsMatch(t, []SDKCredential{key2}, additions) + assert.ElementsMatch(t, []SDKCredential{key1}, expirations) + assert.Empty(t, rotator.DeprecatedCredentials()) +} + +func TestReAnchoringDeprecatedSDKKeyRemovesItFromDeprecatedSet(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + rotator := NewRotator(mockLog.Loggers) + + key1 := config.SDKKey("key1") + key2 := config.SDKKey("key2") + + start := time.Unix(10000, 0) + expiry := start.Add(1 * time.Hour) + + rotator.Initialize([]SDKCredential{key1}) + + // Rotate key1 -> key2 with grace; key1 enters the deprecated set. + rotator.RotateWithGrace(key2, NewGracePeriod(key1, expiry, start)) + rotator.StepTime(start) + assert.ElementsMatch(t, []SDKCredential{key1}, rotator.DeprecatedCredentials()) + + // Re-anchor key2 -> key1 before key1's grace expires. key1 must be promoted out of the + // deprecated set; otherwise the cleanup ticker would later expire the active primary. + rotator.Rotate(key1) + assert.Equal(t, key1, rotator.SDKKey()) + assert.Empty(t, rotator.DeprecatedCredentials()) + + additions, expirations := rotator.StepTime(start) + assert.ElementsMatch(t, []SDKCredential{key1}, additions) + assert.ElementsMatch(t, []SDKCredential{key2}, expirations) + + // Well past the original grace expiry, key1 (the active primary) must NOT be expired. + additions, expirations = rotator.StepTime(expiry.Add(1 * time.Hour)) + assert.Empty(t, additions) + assert.Empty(t, expirations) + assert.Equal(t, key1, rotator.SDKKey()) +} + func TestInitializePopulatesAcceptedSets(t *testing.T) { mockLog := ldlogtest.NewMockLog() rotator := NewRotator(mockLog.Loggers) diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 1b586af3..568a3628 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "slices" "sync" "time" @@ -500,7 +501,29 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env client, err := c.sdkClientFactory(sdkKey, c.sdkConfig, c.sdkInitTimeout) c.mu.Lock() name := c.identifiers.GetDisplayName() + droppedInactive := false + if client != nil && (c.closed || (sdkKey.Defined() && !c.sdkKeyIsActive(sdkKey))) { + // startSDKClient builds the client before taking c.mu, so by the time we hold the lock the key + // may already have been revoked (rotated away) or the environment may have been closed. In + // either case the freshly-built client must be closed here rather than installed, otherwise its + // upstream connection and goroutines leak until env.Close() (and, once closed, nothing ever + // closes it). removeCredential cannot close it because the client was never in c.clients. + // + // The revocation check applies only to a defined key: an undefined (empty) SDK key is never a + // tracked credential -- the rotator filters undefined credentials out of its accepted set -- so + // it can never be "revoked", and dropping its client would break environments that legitimately + // run without an SDK key (e.g. offline or not-yet-configured envs, and test fixtures). + _ = client.Close() + client = nil + droppedInactive = true + } if client != nil { + // If a client already exists for this SDK key (e.g. the key was re-anchored back into the + // primary slot while a previous client for it was still alive in its grace period), close + // the stale one before replacing it so its upstream connection and goroutines are not leaked. + if existing := c.clients[sdkKey]; existing != nil && existing != client { + _ = existing.Close() + } c.clients[sdkKey] = client // The data store instance is created by the SDK when it creates the client. Now that @@ -522,7 +545,8 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env c.initErr = err c.mu.Unlock() - if err != nil { + switch { + case err != nil: if suppressErrors { c.globalLoggers.Warnf("Ignoring error initializing LaunchDarkly client for %q: %+v", name, err) @@ -534,7 +558,13 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env } return } - } else { + case droppedInactive: + // The client initialized successfully but the key was revoked (or the environment was closed) + // before it could be installed, so it was discarded above. The environment is still considered + // ready: it is in a consistent state with no client for this no-longer-tracked key. + c.globalLoggers.Infof("SDK key %s was revoked or the environment was closed before its client "+ + "finished initializing; the client was discarded", sdkKey.Masked()) + default: c.globalLoggers.Infof("Initialized LaunchDarkly client for %q (SDK key %s)", name, sdkKey.Masked()) } if readyCh != nil { @@ -542,6 +572,14 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env } } +// sdkKeyIsActive reports whether the given SDK key is still a tracked credential -- either the primary +// key or one within its deprecation grace period -- according to the rotator. startSDKClient uses this +// to avoid installing (and thereby leaking) a client for a key that was revoked while the client was +// being constructed. +func (c *envContextImpl) sdkKeyIsActive(sdkKey config.SDKKey) bool { + return slices.Contains(c.keyRotator.AllCredentials(), credential.SDKCredential(sdkKey)) +} + func (c *envContextImpl) GetPayloadFilter() config.FilterKey { return c.filterKey } diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 38a2ebde..8b09394f 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -34,6 +34,7 @@ import ( "github.com/launchdarkly/go-sdk-common/v3/ldvalue" ldevents "github.com/launchdarkly/go-sdk-events/v3" "github.com/launchdarkly/go-server-sdk-evaluation/v3/ldbuilders" + ld "github.com/launchdarkly/go-server-sdk/v7" "github.com/launchdarkly/go-server-sdk/v7/ldcomponents" "github.com/launchdarkly/go-server-sdk/v7/subsystems" "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" @@ -506,6 +507,174 @@ func TestNonPrimaryMobileKeyDoesNotStealEventForwarding(t *testing.T) { }) } +// When an SDK key that is still alive in its grace period is re-anchored back into the primary slot, +// a fresh SDK client is started for it. The previously-created client for that same key must be closed +// rather than silently dropped from the clients map, otherwise its upstream connection leaks. +func TestReAnchoringToKeyStillInGraceClosesStaleClient(t *testing.T) { + envConfig := st.EnvMain.Config + keyA := envConfig.SDKKey + keyB := config.SDKKey("keyB") + readyCh := make(chan EnvContext, 1) + + clientCh := make(chan *testclient.FakeLDClient, 10) + clientFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + env := makeBasicEnv(t, envConfig, clientFactory, mockLog.Loggers, readyCh) + defer env.Close() + + assert.Equal(t, env, requireEnvReady(t, readyCh)) + clientA1 := requireClientReady(t, clientCh) + assert.Equal(t, env.GetClient(), clientA1) + + start := time.Unix(1000, 0) + + // Rotate keyA -> keyB, deprecating keyA with an hour-long grace. keyA's client (clientA1) stays + // alive because keyA is still accepted during the grace window. + env.UpdateCredential( + NewCredentialUpdate(keyB). + WithTime(start). + WithGracePeriod(keyA, start.Add(1*time.Hour))) + + clientB := requireClientReady(t, clientCh) + assert.NotEqual(t, clientA1, clientB) + if !helpers.AssertChannelNotClosed(t, clientA1.CloseCh, time.Second, "clientA1 should still be alive during keyA's grace") { + t.FailNow() + } + + // Re-anchor back to keyA while it is still within its grace period. This starts a new client for + // keyA; the stale clientA1 must be closed so its connection is not leaked. + env.UpdateCredential(NewCredentialUpdate(keyA).WithTime(start.Add(10 * time.Minute))) + + clientA2 := requireClientReady(t, clientCh) + assert.NotEqual(t, clientA1, clientA2) + + if !helpers.AssertChannelClosed(t, clientA1.CloseCh, time.Second, "the stale client for keyA should have been closed on re-anchor") { + t.FailNow() + } + // keyB was immediately revoked by the re-anchor, so its client should be closed too. + if !helpers.AssertChannelClosed(t, clientB.CloseCh, time.Second, "client for the revoked keyB should have been closed") { + t.FailNow() + } + + require.Eventually(t, func() bool { + return env.GetClient() == clientA2 + }, time.Second, 10*time.Millisecond, "env.GetClient() should return the new client for keyA after re-anchor") + + creds := env.GetCredentials() + assert.Contains(t, creds, keyA) + assert.NotContains(t, creds, keyB) +} + +// gatedClientFactory wraps the normal fake factory but blocks the factory call for gateKey until +// `gate` is closed, signalling on `started` once that call is in flight. This lets a test interleave +// a credential revocation with an in-flight startSDKClient that has not yet taken c.mu. +func gatedClientFactory( + gate <-chan struct{}, + started chan<- struct{}, + createdCh chan<- *testclient.FakeLDClient, + gateKey config.SDKKey, +) sdks.ClientFactoryFunc { + inner := testclient.FakeLDClientFactoryWithChannel(true, createdCh) + var once sync.Once + return func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == gateKey { + once.Do(func() { close(started) }) + <-gate + } + return inner(sdkKey, cfg, timeout) + } +} + +// When an SDK key is revoked while its client is still being constructed, startSDKClient builds the +// client before taking c.mu, so it can finish and try to install a client for a key that is no longer +// tracked. That client must be closed rather than installed -- otherwise it leaks its upstream +// connection and goroutines, because removeCredential already ran and found nothing in c.clients to +// close, and nothing else will ever close it until env.Close(). +func TestRevokingSDKKeyWhileClientIsStartingDoesNotLeakTheClient(t *testing.T) { + envConfig := st.EnvMain.Config + keyA := envConfig.SDKKey + keyB := config.SDKKey("keyB") + readyCh := make(chan EnvContext, 1) + + clientCh := make(chan *testclient.FakeLDClient, 10) + gate := make(chan struct{}) + started := make(chan struct{}) + factory := gatedClientFactory(gate, started, clientCh, keyA) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) + defer env.Close() + + // Wait until the initial startSDKClient(keyA) is blocked inside the factory, before it takes c.mu. + <-started + + // Revoke keyA by rotating to keyB with no grace. keyA is immediately revoked and removeCredential(keyA) + // runs now -- but c.clients[keyA] is still nil because the initial goroutine is blocked in the factory, + // so nothing is closed and the mapping is simply removed. + env.UpdateCredential(NewCredentialUpdate(keyB).WithTime(time.Unix(1000, 0))) + + creds := env.GetCredentials() + require.NotContains(t, creds, keyA, "keyA should have been revoked") + + // Release the gate; the now-unblocked startSDKClient(keyA) discovers keyA is no longer tracked and + // must close the client it just built rather than installing it. + close(gate) + + // Collect the client that was created for keyA. + var clientA *testclient.FakeLDClient + require.Eventually(t, func() bool { + for { + select { + case c := <-clientCh: + if c.Key == keyA { + clientA = c + } + default: + return clientA != nil + } + } + }, 2*time.Second, 10*time.Millisecond, "expected a client to be created for keyA") + require.NotNil(t, clientA) + + // The client built for the now-revoked keyA must be closed, not leaked. + if !helpers.AssertChannelClosed(t, clientA.CloseCh, time.Second, + "client built for the revoked keyA should have been closed rather than installed") { + t.FailNow() + } + + // And it must not have been installed into the clients map. + impl := env.(*envContextImpl) + impl.mu.Lock() + _, present := impl.clients[keyA] + impl.mu.Unlock() + assert.False(t, present, "revoked keyA should not have a client in the clients map") +} + +// An environment configured without an SDK key (e.g. offline or not-yet-configured envs, and test +// fixtures) must still get its SDK client installed. An undefined key is never a tracked credential, +// so the startup guard that discards clients for revoked keys must not fire for it. +func TestEnvWithoutSDKKeyStillInstallsClient(t *testing.T) { + readyCh := make(chan EnvContext, 1) + clientCh := make(chan *testclient.FakeLDClient, 1) + clientFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + env := makeBasicEnv(t, config.EnvConfig{}, clientFactory, mockLog.Loggers, readyCh) + defer env.Close() + + assert.Equal(t, env, requireEnvReady(t, readyCh)) + client := requireClientReady(t, clientCh) + assert.NotNil(t, client) + assert.Equal(t, client, env.GetClient()) +} + func TestSDKClientCreationFails(t *testing.T) { envConfig := st.EnvWithAllCredentials.Config envConfig.TTL = configtypes.NewOptDuration(time.Hour) From 8b04910a51d20058ea69328c85ba6d3adf279b79 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Thu, 25 Jun 2026 08:51:06 -0700 Subject: [PATCH 20/66] test(relayenv): verify GetClient returns anchor client in multi-key env (#718) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds TestGetClientReturnsAnchorInMultiKeyEnv to lock in the multi-key SDK contract after anchor-only upstream clients: GetClient must return the anchor’s fake upstream client before and after ReconcileCredentials adds two non-anchor keys, must stay non-nil, and must not spawn extra upstream clients. --- internal/relayenv/env_context_impl.go | 11 +++--- internal/relayenv/env_context_impl_test.go | 42 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 568a3628..28be15fa 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -644,10 +644,13 @@ func (c *envContextImpl) GetDeprecatedCredentials() []credential.SDKCredential { func (c *envContextImpl) GetClient() sdks.LDClientContext { c.mu.RLock() defer c.mu.RUnlock() - // In offline mode, there's only one SDK client. This is awkward because we represent the active clients - // as a map, but in this case there's only one client in the map. A refactoring might pull this logic (along with - // differences in add/removeCredential into an interface that is injected based on the environment being - // offline or online. + // c.clients always has at most one entry — the anchor's client. Only the anchor key triggers + // startSDKClient (in addCredential and at construction), so non-anchor server keys never open + // their own upstream connection. + // + // Offline mode uses iteration rather than key-based lookup for historical reasons; both + // approaches are correct because keyRotator is initialized with envConfig.SDKKey before + // startSDKClient is ever called. if c.offline { for _, client := range c.clients { return client diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 8b09394f..3f2d2c62 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -450,6 +450,48 @@ func TestNonAnchorSDKKeysDoNotOpenUpstreamClient(t *testing.T) { } } +// TestGetClientReturnsAnchorInMultiKeyEnv verifies that GetClient returns the anchor's upstream +// client when the environment holds multiple SDK keys. Non-anchor SDK keys share the same +// upstream connection (the anchor's), so GetClient must never return a non-anchor client +// and must remain non-nil after non-anchor keys are added. This is the contract callers of +// GetClient depend on: nil means "env not ready"; non-nil means "use this client." +func TestGetClientReturnsAnchorInMultiKeyEnv(t *testing.T) { + envConfig := st.EnvMain.Config + readyCh := make(chan EnvContext, 1) + clientCh := make(chan *testclient.FakeLDClient, 10) + clientFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + env := makeBasicEnv(t, envConfig, clientFactory, mockLog.Loggers, readyCh) + defer env.Close() + + assert.Equal(t, env, requireEnvReady(t, readyCh)) + anchorClient := requireClientReady(t, clientCh) + assert.Equal(t, envConfig.SDKKey, anchorClient.Key) + + // GetClient must return the anchor's client even before any non-anchor keys are added. + assert.Equal(t, anchorClient, env.GetClient()) + + nonAnchorKey1 := config.SDKKey("non-anchor-key-1") + nonAnchorKey2 := config.SDKKey("non-anchor-key-2") + + env.ReconcileCredentials( + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). + WithPrimarySDKKey(envConfig.SDKKey). + WithSDKKey(nonAnchorKey1). + WithSDKKey(nonAnchorKey2))) + + // No new upstream client was created for the non-anchor keys. + if !helpers.AssertNoMoreValues(t, clientCh, 200*time.Millisecond) { + t.FailNow() + } + + // GetClient still returns the anchor's client — not nil, not a non-anchor client. + assert.Equal(t, anchorClient, env.GetClient()) +} + func TestNonPrimaryMobileKeyDoesNotStealEventForwarding(t *testing.T) { mockLog := ldlogtest.NewMockLog() defer mockLog.DumpIfTestFailed(t) From 287955a4ee70e588f7b07989f6bbadf6068f3287 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Fri, 26 Jun 2026 13:14:50 -0700 Subject: [PATCH 21/66] feat(credentials): wire RAC and offline handlers to ReconcileCredentials (#719) Wires auto-config (RAC) and offline archive handlers to ReconcileCredentials via BuildAcceptedSet, replacing the legacy UpdateCredential / grace-period path. EnvContext drops CredentialUpdate and adds GetSDKKey() / GetMobileKey() so the status endpoint can show a single anchor and primary mobile key deterministically. --- internal/autoconfig/stream_manager.go | 81 +++++++++- .../autoconfig/stream_manager_errors_test.go | 142 +++++++++++++++++- .../stream_manager_test_base_test.go | 57 ++++++- internal/credential/accepted_set.go | 56 +++++-- internal/credential/accepted_set_builder.go | 2 +- internal/credential/accepted_set_test.go | 12 +- internal/credential/rotator.go | 23 ++- internal/credential/rotator_test.go | 12 +- internal/envfactory/env_rep.go | 10 +- internal/envfactory/reconcile_helper.go | 38 +++-- internal/envfactory/reconcile_helper_test.go | 31 +++- internal/relayenv/env_context.go | 53 ++----- internal/relayenv/env_context_impl.go | 17 +-- internal/relayenv/env_context_impl_test.go | 79 ++++++---- .../relayenv/env_context_reanchor_test.go | 10 +- relay/autoconfig_actions.go | 26 ++-- relay/autoconfig_actions_test.go | 5 +- relay/autoconfig_key_change_test.go | 6 +- relay/endpoints_status.go | 19 ++- relay/filedata_actions.go | 39 +++-- relay/filedata_actions_test.go | 15 +- relay/filedata_testdata_test.go | 7 + relay/testutils_test.go | 29 ++-- 23 files changed, 579 insertions(+), 190 deletions(-) diff --git a/internal/autoconfig/stream_manager.go b/internal/autoconfig/stream_manager.go index 335cb8f7..28c25871 100644 --- a/internal/autoconfig/stream_manager.go +++ b/internal/autoconfig/stream_manager.go @@ -388,7 +388,9 @@ func (s *StreamManager) handleStreamEvent(event es.Event) bool { s.cacheCh = nil } putMessage.Data.Persist = true - s.handlePut(putMessage.Data) + if s.handlePut(putMessage.Data) { + shouldRestart = true + } case PatchEvent: var patchMsg PatchMessageData @@ -411,6 +413,13 @@ func (s *StreamManager) handleStreamEvent(event es.Event) bool { s.loggers.Warnf(logMsgEnvHasWrongID, envRep.EnvID, id) break } + // Validate before Upsert so a malformed payload does not advance the version (see + // validateCredentialPayload). Preserve previous state for this env and reconnect. + if err = s.validateCredentialPayload(envRep); err != nil { + s.loggers.Errorf("Received malformed credential payload for environment %q (%s); preserving previous credentials and will restart stream", envRep.EnvID, err) + shouldRestart = true + break + } action := s.envReceiver.Upsert(id, envRep, envRep.Version) s.dispatchEnvAction(config.EnvironmentID(id), envRep, action) if action != ActionNoop { @@ -472,6 +481,7 @@ func (s *StreamManager) handleStreamEvent(event es.Event) bool { return shouldRestart } +// dispatchEnvAction dispatches a single environment action to the handler. func (s *StreamManager) dispatchEnvAction(id config.EnvironmentID, rep envfactory.EnvironmentRep, action Action) { switch action { case ActionNoop: @@ -487,6 +497,20 @@ func (s *StreamManager) dispatchEnvAction(id config.EnvironmentID, rep envfactor } } +// validateCredentialPayload checks that an environment rep carries a structurally valid credential +// set. It is run at the stream parse boundary — before the rep's version is recorded via Upsert — +// mirroring how an unparseable event is handled by gotMalformedEvent. +// +// Per design §9 a malformed credential payload must preserve the previous accepted set and force a +// stream reconnect (RAC is one-way push with no NAK channel, so the reconnect is what makes the +// backend resend a fresh put). Validating here rather than after Upsert is essential: the version is +// not advanced, so the fresh put — which carries the same version — is not deduplicated away by the +// MessageReceiver. Any error from BuildAcceptedSet is a *MalformedCredentialSetError. +func (s *StreamManager) validateCredentialPayload(rep envfactory.EnvironmentRep) error { + _, _, err := envfactory.BuildAcceptedSet(rep.ToParams()) + return err +} + func (s *StreamManager) dispatchFilterAction(id config.FilterID, rep envfactory.FilterRep, action Action) { switch action { case ActionNoop: @@ -509,17 +533,30 @@ func (s *StreamManager) applyCachedContent(content *PutContent) { // All of the private methods below can be assumed to be called from the same goroutine that consumeStream // is on. We will never be processing more than one stream message at the same time. -func (s *StreamManager) handlePut(content PutContent) { +// +// handlePut returns true if the stream should be restarted — a malformed credential payload in any of +// the environments triggers a reconnect (design §9), while still processing the well-formed ones. +func (s *StreamManager) handlePut(content PutContent) bool { // A "put" message represents a full environment set. We will compare them one at a time to the // current set of environments (if any), calling the handler's AddEnvironment for any new ones, // UpdateEnvironment for any that have changed, and DeleteEnvironment for any that are no longer // in the set. + shouldRestart := false + malformedEnvIDs := make(map[config.EnvironmentID]bool) s.loggers.Infof(logMsgPutEvent, len(content.Environments)) for id, rep := range content.Environments { if id != rep.EnvID { s.loggers.Warnf(logMsgEnvHasWrongID, rep.EnvID, id) continue } + // Validate before Upsert so a malformed payload does not advance the version (see + // validateCredentialPayload). Skip this env, preserving its previous state, and reconnect. + if err := s.validateCredentialPayload(rep); err != nil { + s.loggers.Errorf("Received malformed credential payload for environment %q (%s); preserving previous credentials and will restart stream", rep.EnvID, err) + shouldRestart = true + malformedEnvIDs[id] = true + continue + } s.dispatchEnvAction(id, rep, s.envReceiver.Upsert(string(id), rep, rep.Version)) } @@ -545,9 +582,45 @@ func (s *StreamManager) handlePut(content PutContent) { s.handler.ReceivedAllEnvironments() if content.Persist { - if err := s.cache.SetAll(context.Background(), content); err != nil { - s.loggers.Warnf("Failed to write AutoConfig cache: %v", err) + s.persistPut(content, malformedEnvIDs) + } + return shouldRestart +} + +// persistPut writes a put's content to the cache. A clean put is stored atomically with SetAll. When +// the put carried malformed environments, a plain SetAll would corrupt the cache: filtering the +// malformed envs out would drop them, and if every env was malformed it would wipe the cache entirely. +// Instead we keep each malformed env's previously-cached entry and write the resulting snapshot — so +// the valid envs and the filters in this put are persisted, the malformed envs keep their last-good +// value (or are simply omitted when the cache has no prior entry for them), and envs the put removed +// are still dropped. The reconnect a malformed put triggers fetches fresh data for the malformed envs. +// If the prior cache genuinely can't be read, leave it untouched rather than risk dropping entries. +func (s *StreamManager) persistPut(content PutContent, malformedEnvIDs map[config.EnvironmentID]bool) { + if len(malformedEnvIDs) > 0 { + // A nil result with no error means an empty cache (per the Cache contract), not a failure — + // in that case there are simply no prior entries to restore. Only a real read error makes it + // unsafe to rewrite the snapshot, so bail out only then. + prev, err := s.cache.GetAll(context.Background()) + if err != nil { + s.loggers.Warnf("Skipping AutoConfig cache write for a put with malformed credentials (cannot read prior cache): %v", err) + return } + envs := make(map[config.EnvironmentID]envfactory.EnvironmentRep, len(content.Environments)) + for id, rep := range content.Environments { + if malformedEnvIDs[id] { + if prev != nil { + if prevRep, ok := prev.Environments[id]; ok { + envs[id] = prevRep // keep the malformed env's last-good cached entry + } + } + } else { + envs[id] = rep + } + } + content.Environments = envs + } + if err := s.cache.SetAll(context.Background(), content); err != nil { + s.loggers.Warnf("Failed to write AutoConfig cache: %v", err) } } diff --git a/internal/autoconfig/stream_manager_errors_test.go b/internal/autoconfig/stream_manager_errors_test.go index 613dffd3..49353725 100644 --- a/internal/autoconfig/stream_manager_errors_test.go +++ b/internal/autoconfig/stream_manager_errors_test.go @@ -12,6 +12,9 @@ import ( "github.com/launchdarkly/go-sdk-common/v3/ldlog" helpers "github.com/launchdarkly/go-test-helpers/v3" "github.com/launchdarkly/go-test-helpers/v3/httphelpers" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" ) func eventShouldCauseStreamRestart(t *testing.T, event httphelpers.SSEEvent) { @@ -30,6 +33,141 @@ func eventShouldCauseStreamRestart(t *testing.T, event httphelpers.SSEEvent) { }) } +// A credential payload that is valid JSON and a structurally valid event, but whose credential set +// cannot be built (here: an undefined anchor SDK key), must be caught at the parse boundary: the +// previous state is preserved (no AddEnvironment/UpdateEnvironment dispatched) and the stream is +// restarted so the backend resends a fresh put (design §9). This is verified for both patch and put, +// since both paths run the validation before the version is recorded. +func TestMalformedCredentialPayloadCausesStreamRestart(t *testing.T) { + malformedEnv := testEnv1 + malformedEnv.SDKKey = envfactory.SDKKeyRep{Value: config.SDKKey("")} // undefined anchor + + t.Run("patch", func(t *testing.T) { + streamManagerTest(t, nil, func(p streamManagerTestParams) { + p.startStream() + <-p.requestsCh + p.stream.Enqueue(makePatchEnvEvent(malformedEnv)) + select { + case m := <-p.messageHandler.received: + require.Failf(t, "unexpected message", + "must not dispatch for a malformed payload, got %s", m) + case <-p.requestsCh: // reconnect request == stream restart + p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") + case <-time.After(time.Second): + require.Fail(t, "timed out waiting for stream restart") + } + }) + }) + + t.Run("put", func(t *testing.T) { + streamManagerTest(t, nil, func(p streamManagerTestParams) { + p.startStream() + <-p.requestsCh + p.stream.Enqueue(makeEnvPutEvent(malformedEnv)) + // The malformed env is skipped (no add/update); a put still reports ReceivedAllEnvironments, + // which we tolerate. We require that the stream restarts and that no add/update is dispatched. + deadline := time.After(2 * time.Second) + for { + select { + case m := <-p.messageHandler.received: + if m.add != nil || m.update != nil { + require.Failf(t, "unexpected message", + "must not dispatch add/update for a malformed payload, got %s", m) + } + case <-p.requestsCh: // reconnect request == stream restart + p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") + return + case <-deadline: + require.Fail(t, "timed out waiting for stream restart") + } + } + }) + }) +} + +// A put carrying malformed credential payloads must not corrupt the persistent cache: valid envs are +// updated, malformed envs keep their previously-cached entry, and an all-malformed put must not wipe +// the cache (the regression — SetAll is a full replace, so writing a filtered/empty set erased it). +func TestMalformedCredentialPayloadPreservesEnvironmentCache(t *testing.T) { + malformed := func(env envfactory.EnvironmentRep) envfactory.EnvironmentRep { + env.SDKKey = envfactory.SDKKeyRep{Value: config.SDKKey("")} // undefined anchor + return env + } + // updated gives env a distinct SDK key value and a bumped version, so the test can tell a freshly + // persisted update apart from the seeded value. + updated := func(env envfactory.EnvironmentRep, newKey config.SDKKey) envfactory.EnvironmentRep { + env.SDKKey = envfactory.SDKKeyRep{Value: newKey} + env.Version++ + return env + } + seedBothEnvs := func(t *testing.T, p streamManagerTestParams, cache *recordingCache) { + p.stream.Enqueue(makeEnvPutEvent(testEnv1, testEnv2)) + require.Eventually(t, func() bool { + ids := cache.cachedEnvIDs() + return ids[testEnv1.EnvID] && ids[testEnv2.EnvID] + }, time.Second, 10*time.Millisecond, "both envs should be cached after the clean put") + } + + t.Run("valid env is updated, malformed env keeps its previous value", func(t *testing.T) { + cache := &recordingCache{} + streamManagerTestWithCache(t, nil, cache, func(p streamManagerTestParams) { + p.startStream() + <-p.requestsCh + seedBothEnvs(t, p, cache) + + // env1 carries a valid update to a new SDK key; env2 is malformed in the same put. + p.stream.Enqueue(makeEnvPutEvent(updated(testEnv1, "sdkkey1-rotated"), malformed(testEnv2))) + _ = helpers.RequireValue(t, p.requestsCh, time.Second, "timed out waiting for stream restart") + + env1, ok1 := cache.cachedEnv(testEnv1.EnvID) + require.True(t, ok1, "the valid env must remain cached") + assert.Equal(t, config.SDKKey("sdkkey1-rotated"), env1.SDKKey.Value, + "the valid env's update must be persisted") + + env2, ok2 := cache.cachedEnv(testEnv2.EnvID) + require.True(t, ok2, "the malformed env must keep its previously-cached entry, not be dropped") + assert.Equal(t, testEnv2.SDKKey.Value, env2.SDKKey.Value, + "the malformed env must retain its previous (valid) cached value, not the malformed one") + }) + }) + + t.Run("all-malformed put does not wipe the cache", func(t *testing.T) { + cache := &recordingCache{} + streamManagerTestWithCache(t, nil, cache, func(p streamManagerTestParams) { + p.startStream() + <-p.requestsCh + seedBothEnvs(t, p, cache) + + p.stream.Enqueue(makeEnvPutEvent(malformed(testEnv1), malformed(testEnv2))) + _ = helpers.RequireValue(t, p.requestsCh, time.Second, "timed out waiting for stream restart") + + env1, ok1 := cache.cachedEnv(testEnv1.EnvID) + env2, ok2 := cache.cachedEnv(testEnv2.EnvID) + require.True(t, ok1 && ok2, "an all-malformed put must not wipe the cache") + assert.Equal(t, testEnv1.SDKKey.Value, env1.SDKKey.Value, "env1 keeps its previous value") + assert.Equal(t, testEnv2.SDKKey.Value, env2.SDKKey.Value, "env2 keeps its previous value") + }) + }) + + t.Run("mixed put on an empty cache still persists the valid env", func(t *testing.T) { + cache := &recordingCache{} + streamManagerTestWithCache(t, nil, cache, func(p streamManagerTestParams) { + p.startStream() + <-p.requestsCh + // No seed: the cache is empty (GetAll returns nil, not an error). A first put mixing a valid + // and a malformed env must still persist the valid one. + p.stream.Enqueue(makeEnvPutEvent(testEnv1, malformed(testEnv2))) + _ = helpers.RequireValue(t, p.requestsCh, time.Second, "timed out waiting for stream restart") + + env1, ok1 := cache.cachedEnv(testEnv1.EnvID) + require.True(t, ok1, "the valid env must be persisted even on an empty cache") + assert.Equal(t, testEnv1.SDKKey.Value, env1.SDKKey.Value) + _, ok2 := cache.cachedEnv(testEnv2.EnvID) + assert.False(t, ok2, "the malformed env has no prior cached value, so it is omitted") + }) + }) +} + func TestMalformedJSONInEventCausesStreamRestart(t *testing.T) { t.Run("put", func(t *testing.T) { event := httphelpers.SSEEvent{Event: PutEvent, Data: malformedJSON} @@ -89,7 +227,7 @@ func errorShouldCauseReconnect(t *testing.T, errorProducingHandler http.Handler, errorProducingHandler, // first request will get this streamHandler, // request after reconnect will get this ) - streamManagerTestWithStreamHandler(t, handler, stream, func(p streamManagerTestParams) { + streamManagerTestWithStreamHandler(t, handler, stream, noopTestCache{}, func(p streamManagerTestParams) { p.startStream() <-p.requestsCh // first request _ = helpers.RequireValue(t, p.requestsCh, time.Second, "timed out waiting for stream restart") @@ -123,7 +261,7 @@ func TestNoReconnectAfterUnrecoverableHTTPError(t *testing.T) { errorProducingHandler, // first request will get this streamHandler, // request after reconnect will get this ) - streamManagerTestWithStreamHandler(t, handler, stream, func(p streamManagerTestParams) { + streamManagerTestWithStreamHandler(t, handler, stream, noopTestCache{}, func(p streamManagerTestParams) { p.startStream() <-p.requestsCh // first request select { diff --git a/internal/autoconfig/stream_manager_test_base_test.go b/internal/autoconfig/stream_manager_test_base_test.go index 1aff113a..c54e3939 100644 --- a/internal/autoconfig/stream_manager_test_base_test.go +++ b/internal/autoconfig/stream_manager_test_base_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "sync" "testing" "time" @@ -30,6 +31,53 @@ func (noopTestCache) Upsert(context.Context, CacheKind, string, interface{}) err func (noopTestCache) Delete(context.Context, CacheKind, string) error { return nil } func (noopTestCache) Close() error { return nil } +// recordingCache is a minimal in-memory cache: SetAll stores the snapshot and GetAll returns it, so a +// test can drive the read-modify-write path in persistPut and then assert what was persisted. +type recordingCache struct { + mu sync.Mutex + stored *PutContent +} + +func (c *recordingCache) GetAll(context.Context) (*PutContent, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.stored, nil +} +func (c *recordingCache) SetAll(_ context.Context, content PutContent) error { + c.mu.Lock() + defer c.mu.Unlock() + cp := content + c.stored = &cp + return nil +} +func (c *recordingCache) Upsert(context.Context, CacheKind, string, interface{}) error { return nil } +func (c *recordingCache) Delete(context.Context, CacheKind, string) error { return nil } +func (c *recordingCache) Close() error { return nil } + +// cachedEnvIDs returns the set of environment IDs in the most recently stored snapshot. +func (c *recordingCache) cachedEnvIDs() map[config.EnvironmentID]bool { + c.mu.Lock() + defer c.mu.Unlock() + out := make(map[config.EnvironmentID]bool) + if c.stored != nil { + for id := range c.stored.Environments { + out[id] = true + } + } + return out +} + +// cachedEnv returns the stored rep for an environment ID in the most recently stored snapshot. +func (c *recordingCache) cachedEnv(id config.EnvironmentID) (envfactory.EnvironmentRep, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.stored == nil { + return envfactory.EnvironmentRep{}, false + } + rep, ok := c.stored.Environments[id] + return rep, ok +} + const ( testConfigKey config.AutoConfigKey = "test-key" testEnvName = "projname envname" @@ -208,9 +256,13 @@ type testMessageHandler struct { } func streamManagerTest(t *testing.T, initialEvent *httphelpers.SSEEvent, action func(p streamManagerTestParams)) { + streamManagerTestWithCache(t, initialEvent, noopTestCache{}, action) +} + +func streamManagerTestWithCache(t *testing.T, initialEvent *httphelpers.SSEEvent, cache Cache, action func(p streamManagerTestParams)) { streamHandler, stream := httphelpers.SSEHandler(initialEvent) defer stream.Close() - streamManagerTestWithStreamHandler(t, streamHandler, stream, action) + streamManagerTestWithStreamHandler(t, streamHandler, stream, cache, action) } func mustParseURL(t *testing.T, u string) *url.URL { @@ -226,6 +278,7 @@ func streamManagerTestWithStreamHandler( t *testing.T, streamHandler http.Handler, stream httphelpers.SSEStreamControl, + cache Cache, action func(p streamManagerTestParams), ) { mockLog := ldlogtest.NewMockLog() @@ -254,7 +307,7 @@ func streamManagerTestWithStreamHandler( time.Millisecond, rpacProtocolVersion, mockLog.Loggers, - noopTestCache{}, + cache, ) defer p.streamManager.Close() diff --git a/internal/credential/accepted_set.go b/internal/credential/accepted_set.go index c03fd038..8de639b1 100644 --- a/internal/credential/accepted_set.go +++ b/internal/credential/accepted_set.go @@ -54,25 +54,51 @@ func (s AcceptedSet) hasMobileKey(key config.MobileKey) bool { // mistake rather than a benign edge case — surfacing it avoids a silent misconfiguration. var errAcceptedSetMissingSDKKey = errors.New("accepted credential set must contain at least one SDK key") -// MalformedCredentialSetError is returned by AcceptedSetBuilder.Build when the set's designated -// anchor SDK key is missing — a violation of the backend invariant that the anchor (sdkKey.value) -// always appears in sdkKeys[]. Validation happens once, at build time; Rotator.Reconcile trusts the -// set it is handed. +// MalformedCredentialSetError is returned when a credential payload cannot produce a valid +// AcceptedSet. This covers two cases: // -// Because Build rejects the set before it ever reaches Reconcile, the environment's previous accepted -// set is preserved on a malformed payload. The caller is responsible for the second half of the -// malformed-payload policy: reconnecting the RAC stream with jitter to force a fresh put. RAC is -// one-way push with no NAK channel, so without the reconnect the backend would believe the malformed -// patch was applied and would not send fresh state. +// 1. The anchor SDK key (sdkKey.value) is absent or undefined — a violation of the backend +// invariant that an anchor is always designated. +// 2. An entry in sdkKeys[] or mobileKeys[] has an empty value — a credential that would be +// accepted by relay but can never authenticate any SDK. +// +// Validation happens before Reconcile is called; Rotator.Reconcile trusts the set it is handed. +// Because the error is raised before any state mutation, the environment's previous accepted set is +// preserved automatically. The caller is responsible for the second half of the malformed-payload +// policy: reconnecting the RAC stream with jitter to force a fresh put. RAC is one-way push with no +// NAK channel, so without the reconnect the backend would believe the malformed patch was applied +// and would not send fresh state. type MalformedCredentialSetError struct { - // Anchor is the anchor credential that was not found among the set's SDK keys. - Anchor SDKCredential + // msg is the human-readable description set by the constructor. + msg string } func (e *MalformedCredentialSetError) Error() string { - if e.Anchor == nil { - return "malformed credential set: anchor SDK key is missing" + return e.msg +} + +// newMissingAnchorError returns a MalformedCredentialSetError for an absent anchor. +func newMissingAnchorError() *MalformedCredentialSetError { + return &MalformedCredentialSetError{msg: "malformed credential set: anchor SDK key is missing"} +} + +// NewAnchorNotInSetError returns a MalformedCredentialSetError for a payload whose designated anchor +// (sdkKey.value) is defined but not present in the sdkKeys[] array — a structural inconsistency. The +// anchor value is a secret, so it is deliberately not included in the message. +func NewAnchorNotInSetError() *MalformedCredentialSetError { + return &MalformedCredentialSetError{msg: "malformed credential set: anchor SDK key is not present in sdkKeys[]"} +} + +// NewEmptyCredentialError returns a MalformedCredentialSetError for a key-array entry whose +// value field is empty. kind is "sdkKeys" or "mobileKeys"; identifier is the key's identifier +// string (may be empty for old-format payloads that synthesize from the singular fields). +func NewEmptyCredentialError(kind, identifier string) *MalformedCredentialSetError { + if identifier == "" { + return &MalformedCredentialSetError{ + msg: fmt.Sprintf("malformed credential set: %s entry has an empty value", kind), + } + } + return &MalformedCredentialSetError{ + msg: fmt.Sprintf("malformed credential set: %s entry %q has an empty value", kind, identifier), } - return fmt.Sprintf("malformed credential set: anchor SDK key %s is not present in the accepted set", - e.Anchor.Masked()) } diff --git a/internal/credential/accepted_set_builder.go b/internal/credential/accepted_set_builder.go index 8e82189d..7a3f0968 100644 --- a/internal/credential/accepted_set_builder.go +++ b/internal/credential/accepted_set_builder.go @@ -106,7 +106,7 @@ func (b *AcceptedSetBuilder) Build() (AcceptedSet, error) { return AcceptedSet{}, errAcceptedSetMissingSDKKey } if !b.set.primarySdkKey.Defined() { - return AcceptedSet{}, &MalformedCredentialSetError{Anchor: nil} + return AcceptedSet{}, newMissingAnchorError() } return b.set, nil } diff --git a/internal/credential/accepted_set_test.go b/internal/credential/accepted_set_test.go index dec56d34..4519b1ee 100644 --- a/internal/credential/accepted_set_test.go +++ b/internal/credential/accepted_set_test.go @@ -9,11 +9,13 @@ import ( ) func TestMalformedCredentialSetErrorMessage(t *testing.T) { - // A nil anchor reports "missing" rather than dereferencing a nil credential. + // Missing anchor. assert.Equal(t, "malformed credential set: anchor SDK key is missing", - (&MalformedCredentialSetError{Anchor: nil}).Error()) + newMissingAnchorError().Error()) - // A defined anchor is masked in the message. - assert.Contains(t, (&MalformedCredentialSetError{Anchor: config.SDKKey("sdk-abcd1234")}).Error(), - "...1234") + // Empty credential value. + assert.Contains(t, NewEmptyCredentialError("sdkKeys", "my-key").Error(), "empty value") + + // The config.SDKKey import is exercised; confirm it still compiles. + _ = config.SDKKey("sdk-abcd1234") } diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 62a95dc6..c495282b 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -151,11 +151,30 @@ func (r *Rotator) deprecatedCredentials() []SDKCredential { return deprecated } -// DeprecatedCredentials returns deprecated credentials (not expired or primary.) +// DeprecatedCredentials returns the SDK keys being phased out — every accepted SDK key, other than the +// anchor, that carries a future expiry. (Per-key expiry is stored as data on the accepted entry; the +// cleanup ticker drops the key once it elapses.) EnvContext.GetDeprecatedCredentials delegates here to +// populate the status endpoint's expiringSdkKey field. +// +// Mobile keys are deliberately not returned even though they expire the same way SDK keys do — carried +// as per-key expiry and dropped by the same cleanup ticker. They are omitted only because the status +// endpoint has no expiringMobileKey field to populate, not because mobile-key expiry is unimplemented. func (r *Rotator) DeprecatedCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() - return r.deprecatedCredentials() + + // TEMPORARY (legacy rotation path): keys deprecated via RotateWithGrace live in the + // deprecatedSdkKeys / deprecatedMobileKeys buckets, which the reconcile path never populates. Once + // the legacy path is removed (SDK-2603) these buckets are always empty; delete this line and the + // deprecatedCredentials helper, leaving only the accepted-with-expiry logic below. + out := r.deprecatedCredentials() + + for key, info := range r.acceptedSDKKeys { + if info.expiry != nil && key != r.primarySdkKey { + out = append(out, key) + } + } + return out } // AllCredentials returns the primary and deprecated credentials as one list. diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 3e0c4acc..b5e7a67e 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -542,8 +542,9 @@ func TestReconcileRevokesOmittedKeys(t *testing.T) { func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { // Reconcile stores per-key expiry as data on the accepted entry; before that expiry passes, an - // expiring key is simply accepted and non-deprecated. The cleanup ticker (StepTime) only acts on - // the expiry once it elapses — see TestReconcileExpiringKeysAreEvictedByStepTime. + // expiring key is still accepted (it authenticates and appears in PrimaryCredentials) while also + // being reported as deprecated — accepted, but on its way out. The cleanup ticker (StepTime) only + // drops it once the expiry elapses — see TestReconcileExpiringKeysAreEvictedByStepTime. r := newTestRotator() anchor := config.SDKKey("anchor") expiringSDK := config.SDKKey("expiring-sdk") @@ -562,9 +563,12 @@ func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { assert.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, additions) assert.Empty(t, expirations) - // All keys are accepted and non-deprecated in the foundation. + // Every key is accepted (still authenticates)... assert.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, r.PrimaryCredentials()) - assert.Empty(t, r.DeprecatedCredentials()) + // ...and the non-anchor SDK key carrying an expiry is also reported as deprecated (being phased + // out). The expiring mobile key is not: there is no expiringMobileKey status field, so the reconcile + // path treats it as accepted-only. + assert.ElementsMatch(t, []SDKCredential{expiringSDK}, r.DeprecatedCredentials()) } func TestReconcilePrimaryMobileKeyIsAlwaysAccepted(t *testing.T) { diff --git a/internal/envfactory/env_rep.go b/internal/envfactory/env_rep.go index ace2a54f..151f93a2 100644 --- a/internal/envfactory/env_rep.go +++ b/internal/envfactory/env_rep.go @@ -183,8 +183,14 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { params.AcceptedMobileKeys = append(params.AcceptedMobileKeys, entry) } } else { - // Old-format payload: synthesize from the singular mobKey field. - params.AcceptedMobileKeys = []AcceptedMobileKey{{Value: r.MobKey}} + // Old-format payload: synthesize from the singular mobKey field. An undefined mobKey means the + // environment has no mobile key (e.g. a server-side-only environment) — leave the set empty + // rather than synthesizing a phantom empty-value entry, which BuildAcceptedSet would otherwise + // reject as a malformed credential. + params.AcceptedMobileKeys = []AcceptedMobileKey{} + if r.MobKey.Defined() { + params.AcceptedMobileKeys = append(params.AcceptedMobileKeys, AcceptedMobileKey{Value: r.MobKey}) + } } return params diff --git a/internal/envfactory/reconcile_helper.go b/internal/envfactory/reconcile_helper.go index db9a8781..efa94ca0 100644 --- a/internal/envfactory/reconcile_helper.go +++ b/internal/envfactory/reconcile_helper.go @@ -20,11 +20,11 @@ import ( // The builder de-duplicates by value, so an anchor or primary mobile key that also appears in its // array is added only once. // -// If no anchor is designated — params.SDKKey is undefined — no set is built and a -// *credential.MalformedCredentialSetError is returned with an empty AcceptedSet. The caller must +// A *credential.MalformedCredentialSetError is returned (with an empty AcceptedSet) for a +// structurally malformed payload: an undefined anchor (params.SDKKey not set), a defined anchor that +// is absent from params.AcceptedSDKKeys, or an array entry with an empty value. The caller must // preserve the previous accepted state and, for RAC handlers, reconnect the stream with jitter to -// force a fresh put. Structural validation of the wire payload (undefined credentials, an anchor -// absent from the array) happens upstream when the payload is parsed into params. +// force a fresh put. This is the single home for the anchor invariant. func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config.SDKKey, error) { anchor := params.SDKKey @@ -36,12 +36,21 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. WithPrimarySDKKey(anchor). WithPrimaryMobileKey(params.MobileKey) - // Add the remaining accepted keys. The builder de-duplicates by value, so the anchor and the - // primary mobile key — already added permanently above — are ignored when they reappear in - // their arrays. That also defends the anchor-never-expiring invariant: a payload that (wrongly) - // carries an expiry on the anchor's own entry cannot demote it, because the permanent anchor is - // already present. + // Validate and add the remaining accepted keys. The builder de-duplicates by value, so the + // anchor and the primary mobile key — already added permanently above — are ignored when they + // reappear in their arrays. That also defends the anchor-never-expiring invariant: a payload + // that (wrongly) carries an expiry on the anchor's own entry cannot demote it. + // + // Entries with an empty value are structurally malformed: relay would silently accept them but + // they can never authenticate any SDK. Reject loudly rather than produce a credential-short env. + anchorInArray := false for _, k := range params.AcceptedSDKKeys { + if !k.Value.Defined() { + return credential.AcceptedSet{}, anchor, credential.NewEmptyCredentialError("sdkKeys", k.Key) + } + if k.Value == anchor { + anchorInArray = true + } if k.Expiry.IsZero() { b.WithSDKKey(k.Value) } else { @@ -49,7 +58,18 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. } } + // The anchor must be one of the accepted SDK keys: the backend lists it in sdkKeys[] (and ToParams + // synthesizes it into the array for old-format payloads). A defined anchor absent from the array is + // a structurally malformed payload — reject it per §9 rather than letting WithPrimarySDKKey above + // silently synthesize it into the set. + if anchor.Defined() && !anchorInArray { + return credential.AcceptedSet{}, anchor, credential.NewAnchorNotInSetError() + } + for _, k := range params.AcceptedMobileKeys { + if !k.Value.Defined() { + return credential.AcceptedSet{}, anchor, credential.NewEmptyCredentialError("mobileKeys", k.Key) + } if k.Expiry.IsZero() { b.WithMobileKey(k.Value) } else { diff --git a/internal/envfactory/reconcile_helper_test.go b/internal/envfactory/reconcile_helper_test.go index 2c244711..0f9b7711 100644 --- a/internal/envfactory/reconcile_helper_test.go +++ b/internal/envfactory/reconcile_helper_test.go @@ -156,6 +156,10 @@ func TestBuildAcceptedSet_Deexpiry(t *testing.T) { // longer rejected here: WithPrimarySDKKey adds and designates the anchor regardless, so the // resulting set contains both the anchor and the array entry. Structural validation of the wire // payload (anchor-absent-from-array) happens upstream when the payload is parsed into params. +// TestBuildAcceptedSet_AnchorNotInArray verifies that a defined anchor absent from the sdkKeys[] array +// yields a *credential.MalformedCredentialSetError per design §9: the payload is structurally +// inconsistent (the designated primary is not in the authoritative array), so it must be rejected +// rather than silently synthesized into the set. func TestBuildAcceptedSet_AnchorNotInArray(t *testing.T) { params := makeParams( "sdk-anchor", @@ -164,16 +168,31 @@ func TestBuildAcceptedSet_AnchorNotInArray(t *testing.T) { }, "mob-primary", ) - set, anchor, err := BuildAcceptedSet(params) + _, _, err := BuildAcceptedSet(params) + + require.Error(t, err) + var malformed *credential.MalformedCredentialSetError + require.True(t, errors.As(err, &malformed)) + assert.Contains(t, malformed.Error(), "not present in sdkKeys[]") +} + +// TestBuildAcceptedSet_NoMobileKey verifies that an environment with no mobile key (e.g. a +// server-side-only environment) is valid: ToParams must not synthesize a phantom empty mobileKeys +// entry that BuildAcceptedSet would reject as malformed. +func TestBuildAcceptedSet_NoMobileKey(t *testing.T) { + rep := EnvironmentRep{ + EnvID: "env-abc", + SDKKey: SDKKeyRep{Value: config.SDKKey("sdk-anchor")}, + // no MobKey, no MobileKeys + } + set, anchor, err := BuildAcceptedSet(rep.ToParams()) require.NoError(t, err) assert.Equal(t, config.SDKKey("sdk-anchor"), anchor) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithPrimarySDKKey("sdk-anchor"). // added + designated even though absent from the array - WithSDKKey("sdk-other"). - WithPrimaryMobileKey("mob-primary")) + WithPrimarySDKKey("sdk-anchor")) assert.Equal(t, expected, set) } @@ -192,10 +211,6 @@ func TestBuildAcceptedSet_AnchorUndefined(t *testing.T) { require.Error(t, err) var malformed *credential.MalformedCredentialSetError require.True(t, errors.As(err, &malformed)) - // An undefined anchor must produce the "missing" message, not the "not present" one. This - // only holds if Anchor is an untyped nil — a boxed zero-value config.SDKKey would be non-nil - // and route Error() down the wrong branch. - assert.Nil(t, malformed.Anchor) assert.Contains(t, malformed.Error(), "anchor SDK key is missing") } diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index fbfcd357..61073b19 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -21,43 +21,6 @@ import ( ldeval "github.com/launchdarkly/go-server-sdk-evaluation/v3" ) -// CredentialUpdate specifies the primary credential of a given credential kind for an environment. -// For example, an environment may have a primary SDK key and a primary mobile key at the same time; each would -// be specified in individual CredentialUpdate objects. -type CredentialUpdate struct { - // The new primary credential - primary credential.SDKCredential - // An optional deprecated credential (only SDK keys are supported currently) - deprecated config.SDKKey - // When the deprecated credential expires - expiry time.Time - // The current time - now time.Time -} - -// NewCredentialUpdate creates a CredentialUpdate from a given primary credential. -// The default behavior of the environment is to immediately revoke the previous credential of this kind. -func NewCredentialUpdate(primary credential.SDKCredential) *CredentialUpdate { - return &CredentialUpdate{primary: primary, now: time.Now()} -} - -// WithGracePeriod modifies the default behavior from immediate revocation to a delayed revocation of the previous -// credential. During the grace period, the previous credential continues to function. -func (c *CredentialUpdate) WithGracePeriod(deprecated config.SDKKey, expiry time.Time) *CredentialUpdate { - c.deprecated = deprecated - c.expiry = expiry - return c -} - -// WithTime overrides the update's current time for testing purposes. -// Because the environment's credential rotation algorithm compares the current time to the specific expiry of -// each credential, this can be used to trigger behavior in a more predictable way than relying on the actual time -// in the test. -func (c *CredentialUpdate) WithTime(t time.Time) *CredentialUpdate { - c.now = t - return c -} - // EnvContext is the interface for all Relay operations that are specific to one configured LD environment. // // The EnvContext is normally associated with an LDClient instance from the Go SDK, and allows direct access @@ -77,13 +40,15 @@ type EnvContext interface { // SetIdentifiers updates the environment and project names and keys. SetIdentifiers(EnvIdentifiers) - // UpdateCredential updates the environment with a new credential, optionally deprecating a previous one - // with a grace period. - // - // This is the legacy single-credential rotation API. It is retained while the action handlers still - // drive rotation through it; the full-set ReconcileCredentials below will take over once the handlers - // are migrated to it. - UpdateCredential(update *CredentialUpdate) + // GetSDKKey returns the anchor SDK key — the primary key that owns the upstream connection. + // Use this when you need exactly the anchor (e.g. the status endpoint's sdkKey field) rather + // than the full accepted set returned by GetCredentials. + GetSDKKey() config.SDKKey + + // GetMobileKey returns the primary (default) mobile key. Like GetSDKKey for the anchor, use this + // for the status endpoint's mobileKey field rather than iterating GetCredentials, which may return + // several accepted mobile keys (primary + expiring) in nondeterministic order. + GetMobileKey() config.MobileKey // ReconcileCredentials atomically reconciles the environment's accepted credentials to match // newSet. The set names its own anchor (the SDK key that owns the upstream connection) and diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 28be15fa..7f9516e6 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -598,15 +598,6 @@ func (c *envContextImpl) SetIdentifiers(ei EnvIdentifiers) { c.identifiers = ei } -func (c *envContextImpl) UpdateCredential(update *CredentialUpdate) { - if !update.deprecated.Defined() { - c.keyRotator.Rotate(update.primary) - } else { - c.keyRotator.RotateWithGrace(update.primary, credential.NewGracePeriod(update.deprecated, update.expiry, update.now)) - } - c.triggerCredentialChanges(update.now) -} - func (c *envContextImpl) ReconcileCredentials(newSet credential.AcceptedSet) { c.reconcileCredentials(newSet, time.Now()) } @@ -637,6 +628,14 @@ func (c *envContextImpl) GetCredentials() []credential.SDKCredential { return c.keyRotator.PrimaryCredentials() } +func (c *envContextImpl) GetSDKKey() config.SDKKey { + return c.keyRotator.SDKKey() +} + +func (c *envContextImpl) GetMobileKey() config.MobileKey { + return c.keyRotator.MobileKey() +} + func (c *envContextImpl) GetDeprecatedCredentials() []credential.SDKCredential { return c.keyRotator.DeprecatedCredentials() } diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 3f2d2c62..3368cfa3 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -279,18 +279,26 @@ func TestChangeSDKKey(t *testing.T) { assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) assert.Empty(t, env.GetDeprecatedCredentials()) - // For the purposes of key rotation, we'll make time deterministic. We drive the grace-period - // setup through the legacy UpdateCredential API (with an injected time), then advance the cleanup - // ticker (triggerCredentialChanges) to expire the deprecated key — the same path the periodic - // ticker uses in production. + // For the purposes of key rotation, we'll make time deterministic. We build an AcceptedSet + // with key2 as anchor and envConfig.SDKKey expiring in one hour, then drive the time-injectable + // reconcileCredentials. The cleanup ticker path (triggerCredentialChanges) is exercised below. start := time.Unix(1000, 0) // Upon rotating to key2, the original key should still be valid for an hour. - envImpl.UpdateCredential(NewCredentialUpdate(key2). - WithGracePeriod(envConfig.SDKKey, start.Add(1 * time.Hour)). - WithTime(start)) + rotationSet, err := credential.NewAcceptedSetBuilder(). + WithPrimarySDKKey(key2). + WithExpiringSDKKey(envConfig.SDKKey, start.Add(1*time.Hour)). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(rotationSet, start) - assert.Equal(t, []credential.SDKCredential{key2}, env.GetCredentials()) + // In the new accepted-set model both keys are accepted (GetCredentials includes both) until + // the old key's expiry elapses. GetDeprecatedCredentials reports the expiring accepted key so + // callers like the status endpoint can surface it without distinguishing the rotation path. + creds := env.GetCredentials() + assert.Len(t, creds, 2) + assert.Contains(t, creds, key2) + assert.Contains(t, creds, envConfig.SDKKey) assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetDeprecatedCredentials()) client2 := requireClientReady(t, clientCh) @@ -306,14 +314,14 @@ func TestChangeSDKKey(t *testing.T) { t.FailNow() } - // Simulate an amount of time passing that is less than the deprecation period. The original key should still be valid. + // Simulate an amount of time passing that is less than the expiry window. The original key should still be valid. envImpl.triggerCredentialChanges(start.Add(45 * time.Minute)) if !helpers.AssertChannelNotClosed(t, client1.CloseCh, 1*time.Second, "client for envConfig.SDKKey should not have been closed yet") { t.FailNow() } - // We are now an instant after the deprecation period. This should cause the original key to become expired - // and trigger the client to close. + // We are now an instant after the expiry. This should cause the original key to be removed + // and trigger its client to close. envImpl.triggerCredentialChanges(start.Add(1*time.Hour + 1*time.Millisecond)) assert.Equal(t, []credential.SDKCredential{key2}, env.GetCredentials()) assert.Empty(t, env.GetDeprecatedCredentials()) @@ -552,7 +560,12 @@ func TestNonPrimaryMobileKeyDoesNotStealEventForwarding(t *testing.T) { // When an SDK key that is still alive in its grace period is re-anchored back into the primary slot, // a fresh SDK client is started for it. The previously-created client for that same key must be closed // rather than silently dropped from the clients map, otherwise its upstream connection leaks. -func TestReAnchoringToKeyStillInGraceClosesStaleClient(t *testing.T) { +// Originally a regression test from #716 for the old UpdateCredential path, where re-anchoring to a +// key still in its grace period spawned a fresh client and orphaned the old one. Under the +// ReconcileCredentials model that leak is structurally impossible: re-anchoring to a still-accepted key +// emits no "addition", so its existing client is reused rather than re-spawned, and the displaced +// anchor's client is closed by removeCredential. This test now verifies that reuse-and-no-leak guarantee. +func TestReAnchoringToKeyStillInGraceReusesItsClient(t *testing.T) { envConfig := st.EnvMain.Config keyA := envConfig.SDKKey keyB := config.SDKKey("keyB") @@ -575,10 +588,11 @@ func TestReAnchoringToKeyStillInGraceClosesStaleClient(t *testing.T) { // Rotate keyA -> keyB, deprecating keyA with an hour-long grace. keyA's client (clientA1) stays // alive because keyA is still accepted during the grace window. - env.UpdateCredential( - NewCredentialUpdate(keyB). - WithTime(start). - WithGracePeriod(keyA, start.Add(1*time.Hour))) + env.(*envContextImpl).reconcileCredentials( + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). + WithPrimarySDKKey(keyB). + WithExpiringSDKKey(keyA, start.Add(1*time.Hour))), + start) clientB := requireClientReady(t, clientCh) assert.NotEqual(t, clientA1, clientB) @@ -586,24 +600,31 @@ func TestReAnchoringToKeyStillInGraceClosesStaleClient(t *testing.T) { t.FailNow() } - // Re-anchor back to keyA while it is still within its grace period. This starts a new client for - // keyA; the stale clientA1 must be closed so its connection is not leaked. - env.UpdateCredential(NewCredentialUpdate(keyA).WithTime(start.Add(10 * time.Minute))) - - clientA2 := requireClientReady(t, clientCh) - assert.NotEqual(t, clientA1, clientA2) + // Re-anchor back to keyA while it is still within its grace period. Because keyA is still an accepted + // credential, its existing client (clientA1) is reused as the anchor client rather than a new one + // being started -- so there is no stale client to orphan. keyB is omitted from the set (no expiry), + // so it is revoked immediately and its client is closed. + env.(*envContextImpl).reconcileCredentials( + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithPrimarySDKKey(keyA)), + start.Add(10*time.Minute)) - if !helpers.AssertChannelClosed(t, clientA1.CloseCh, time.Second, "the stale client for keyA should have been closed on re-anchor") { + // keyB was revoked by the re-anchor, so its client is closed. + if !helpers.AssertChannelClosed(t, clientB.CloseCh, time.Second, "client for the revoked keyB should have been closed") { t.FailNow() } - // keyB was immediately revoked by the re-anchor, so its client should be closed too. - if !helpers.AssertChannelClosed(t, clientB.CloseCh, time.Second, "client for the revoked keyB should have been closed") { + // clientA1 is reused, not closed or churned: re-anchoring to a still-accepted key must not tear down + // its working upstream connection. + if !helpers.AssertChannelNotClosed(t, clientA1.CloseCh, time.Second, "clientA1 should be reused as the anchor client, not closed") { + t.FailNow() + } + // No new client is started for keyA -- the existing one is reused. + if !helpers.AssertNoMoreValues(t, clientCh, time.Second, "re-anchoring to an in-grace key must not start a new client") { t.FailNow() } require.Eventually(t, func() bool { - return env.GetClient() == clientA2 - }, time.Second, 10*time.Millisecond, "env.GetClient() should return the new client for keyA after re-anchor") + return env.GetClient() == clientA1 + }, time.Second, 10*time.Millisecond, "env.GetClient() should return the reused client for keyA after re-anchor") creds := env.GetCredentials() assert.Contains(t, creds, keyA) @@ -658,7 +679,9 @@ func TestRevokingSDKKeyWhileClientIsStartingDoesNotLeakTheClient(t *testing.T) { // Revoke keyA by rotating to keyB with no grace. keyA is immediately revoked and removeCredential(keyA) // runs now -- but c.clients[keyA] is still nil because the initial goroutine is blocked in the factory, // so nothing is closed and the mapping is simply removed. - env.UpdateCredential(NewCredentialUpdate(keyB).WithTime(time.Unix(1000, 0))) + env.(*envContextImpl).reconcileCredentials( + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithPrimarySDKKey(keyB)), + time.Unix(1000, 0)) creds := env.GetCredentials() require.NotContains(t, creds, keyA, "keyA should have been revoked") diff --git a/internal/relayenv/env_context_reanchor_test.go b/internal/relayenv/env_context_reanchor_test.go index e6078a05..ff05b693 100644 --- a/internal/relayenv/env_context_reanchor_test.go +++ b/internal/relayenv/env_context_reanchor_test.go @@ -24,6 +24,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/basictypes" + "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/bigsegments" "github.com/launchdarkly/ld-relay/v8/internal/httpconfig" "github.com/launchdarkly/ld-relay/v8/internal/sdks" @@ -101,9 +102,12 @@ func (f *sharedStoreFactory) Build(_ subsystems.ClientContext) (subsystems.DataS // reconcileCredentials directly so the grace-period math is deterministic. func reanchor(t *testing.T, env EnvContext, newKey, oldKey config.SDKKey, now time.Time) { t.Helper() - env.(*envContextImpl).UpdateCredential(NewCredentialUpdate(newKey). - WithGracePeriod(oldKey, now.Add(time.Hour)). - WithTime(now)) + set, err := credential.NewAcceptedSetBuilder(). + WithPrimarySDKKey(newKey). + WithExpiringSDKKey(oldKey, now.Add(time.Hour)). + Build() + require.NoError(t, err) + env.(*envContextImpl).reconcileCredentials(set, now) } // ----------------------------------------------------------------------------------------------- diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index 57625701..f8ea7638 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -3,7 +3,6 @@ package relay import ( "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" - "github.com/launchdarkly/ld-relay/v8/internal/relayenv" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" ) @@ -34,10 +33,12 @@ func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentPar return } - if params.ExpiringSDKKey.Defined() { - update := relayenv.NewCredentialUpdate(params.SDKKey) - env.UpdateCredential(update.WithGracePeriod(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration)) + set, _, buildErr := envfactory.BuildAcceptedSet(params) + if buildErr != nil { + a.r.loggers.Errorf(logMsgAutoConfEnvInitError, params.Identifiers.GetDisplayName(), buildErr) + return } + env.ReconcileCredentials(set) } func (a *relayAutoConfigActions) UpdateEnvironment(params envfactory.EnvironmentParams) { @@ -51,16 +52,15 @@ func (a *relayAutoConfigActions) UpdateEnvironment(params envfactory.Environment env.SetTTL(params.TTL) env.SetSecureMode(params.SecureMode) - if params.MobileKey.Defined() { - env.UpdateCredential(relayenv.NewCredentialUpdate(params.MobileKey)) - } - if params.SDKKey.Defined() { - update := relayenv.NewCredentialUpdate(params.SDKKey) - if params.ExpiringSDKKey.Defined() { - update = update.WithGracePeriod(params.ExpiringSDKKey.Key, params.ExpiringSDKKey.Expiration) - } - env.UpdateCredential(update) + set, _, buildErr := envfactory.BuildAcceptedSet(params) + if buildErr != nil { + // Credential payloads are validated at the stream parse boundary (see StreamManager) before + // being dispatched here, so a malformed set should not reach this point. Log defensively and + // preserve the previous credentials rather than applying a partial set. + a.r.loggers.Errorf(logMsgAutoConfEnvInitError, params.Identifiers.GetDisplayName(), buildErr) + return } + env.ReconcileCredentials(set) } func (a *relayAutoConfigActions) DeleteEnvironment(id config.EnvironmentID, filter config.FilterKey) { diff --git a/relay/autoconfig_actions_test.go b/relay/autoconfig_actions_test.go index 2ae5ce16..dc2c7fbb 100644 --- a/relay/autoconfig_actions_test.go +++ b/relay/autoconfig_actions_test.go @@ -227,7 +227,8 @@ func TestAutoConfigAddEnvironmentWithExpiringSDKKey(t *testing.T) { env := p.awaitEnvironment(envWithKeys.id) assertEnvProps(t, envWithKeys.params(), env) - expectedCredentials := credentialsAsSet(envWithKeys.id, envWithKeys.mobKey, envWithKeys.SDKKey()) + // Both the new anchor key and the expiring old key are in the accepted set until oldKey expires. + expectedCredentials := credentialsAsSet(envWithKeys.id, envWithKeys.mobKey, newKey, oldKey) assert.Equal(t, expectedCredentials, credentialsAsSet(env.GetCredentials()...)) paramsWithOldKey := envWithKeys.params() @@ -236,7 +237,7 @@ func TestAutoConfigAddEnvironmentWithExpiringSDKKey(t *testing.T) { }) } -// When addEnvironment fails, the auto-config handler must not go on to call UpdateCredential on +// When addEnvironment fails, the auto-config handler must not go on to call ReconcileCredentials on // the nil EnvContext it got back. This is only reachable when the payload also carries an expiring // SDK key (the gate that triggers the credential update). We force the failure deterministically by // closing the Relay first, so addEnvironment returns errAlreadyClosed with a nil env. diff --git a/relay/autoconfig_key_change_test.go b/relay/autoconfig_key_change_test.go index 2f0e29b7..0e95dd39 100644 --- a/relay/autoconfig_key_change_test.go +++ b/relay/autoconfig_key_change_test.go @@ -199,7 +199,7 @@ func TestAutoConfigRemovesCredentialForExpiredSDKKey(t *testing.T) { assert.Equal(t, modified.SDKKey(), client2.Key) p.awaitCredentialsUpdated(env, modified.params()) - newCredentials := credentialsAsSet(env.GetCredentials()...) + // Before expiry, old key is still accepted (it's in the accepted set with a countdown). foundEnvWithOldKey, _ := p.relay.getEnvironment(sdkauth.New(oldKey)) assert.Equal(t, env, foundEnvWithOldKey) @@ -207,7 +207,9 @@ func TestAutoConfigRemovesCredentialForExpiredSDKKey(t *testing.T) { t.FailNow() } - assert.Equal(t, newCredentials, credentialsAsSet(env.GetCredentials()...)) + // After expiry, old key is removed; new key + mobile key + env ID are the only credentials left. + expectedAfterExpiry := credentialsAsSet(modified.id, modified.mobKey, modified.SDKKey()) + assert.Equal(t, expectedAfterExpiry, credentialsAsSet(env.GetCredentials()...)) noEnv, _ := p.relay.getEnvironment(sdkauth.New(oldKey)) assert.Nil(t, noEnv) }) diff --git a/relay/endpoints_status.go b/relay/endpoints_status.go index b0ee6340..01546a7b 100644 --- a/relay/endpoints_status.go +++ b/relay/endpoints_status.go @@ -46,17 +46,20 @@ func statusHandler(relay *Relay) http.Handler { ProjName: identifiers.ProjName, } + // Use the anchor SDK key and primary mobile key specifically — GetCredentials() may return + // multiple SDK and mobile keys (primary + expiring), so iterating it for these singular + // status fields would give a non-deterministic result. + if key := clientCtx.GetSDKKey(); key.Defined() { + status.SDKKey = sdks.ObscureKey(string(key)) + } + if key := clientCtx.GetMobileKey(); key.Defined() { + status.MobileKey = sdks.ObscureKey(string(key)) + } for _, c := range clientCtx.GetCredentials() { - switch c := c.(type) { - case config.SDKKey: - status.SDKKey = sdks.ObscureKey(string(c)) - case config.MobileKey: - status.MobileKey = sdks.ObscureKey(string(c)) - case config.EnvironmentID: - status.EnvID = string(c) + if envID, ok := c.(config.EnvironmentID); ok { + status.EnvID = string(envID) } } - for _, c := range clientCtx.GetDeprecatedCredentials() { if key, ok := c.(config.SDKKey); ok { status.ExpiringSDKKey = sdks.ObscureKey(string(key)) diff --git a/relay/filedata_actions.go b/relay/filedata_actions.go index 1528ccd1..24486e8f 100644 --- a/relay/filedata_actions.go +++ b/relay/filedata_actions.go @@ -1,9 +1,10 @@ package relay import ( + "errors" "time" - "github.com/launchdarkly/ld-relay/v8/internal/relayenv" + "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" @@ -56,10 +57,21 @@ func (a *relayFileDataActions) AddEnvironment(ae filedata.ArchiveEnvironment) { a.r.loggers.Errorf(logMsgAutoConfEnvInitError, ae.Params.Identifiers.GetDisplayName(), err) return } - if ae.Params.ExpiringSDKKey.Defined() { - update := relayenv.NewCredentialUpdate(ae.Params.SDKKey) - env.UpdateCredential(update.WithGracePeriod(ae.Params.ExpiringSDKKey.Key, ae.Params.ExpiringSDKKey.Expiration)) + + set, _, buildErr := envfactory.BuildAcceptedSet(ae.Params) + if buildErr != nil { + var malformed *credential.MalformedCredentialSetError + if errors.As(buildErr, &malformed) { + a.r.loggers.Errorf("Malformed credential payload for offline environment %q — preserving previous credentials: %s", ae.Params.Identifiers.GetDisplayName(), buildErr) + } else { + a.r.loggers.Errorf(logMsgAutoConfEnvInitError, ae.Params.Identifiers.GetDisplayName(), buildErr) + } + // No reconnect for offline mode: preserve previous state (env was just created with + // the singular sdkKey from envConfig) and wait for the next archive reload. + } else { + env.ReconcileCredentials(set) } + select { case updates := <-updatesCh: if a.envUpdates == nil { @@ -89,15 +101,18 @@ func (a *relayFileDataActions) UpdateEnvironment(ae filedata.ArchiveEnvironment) env.SetTTL(ae.Params.TTL) env.SetSecureMode(ae.Params.SecureMode) - if ae.Params.MobileKey.Defined() { - env.UpdateCredential(relayenv.NewCredentialUpdate(ae.Params.MobileKey)) - } - if ae.Params.SDKKey.Defined() { - update := relayenv.NewCredentialUpdate(ae.Params.SDKKey) - if ae.Params.ExpiringSDKKey.Defined() { - update = update.WithGracePeriod(ae.Params.ExpiringSDKKey.Key, ae.Params.ExpiringSDKKey.Expiration) + set, _, buildErr := envfactory.BuildAcceptedSet(ae.Params) + if buildErr != nil { + var malformed *credential.MalformedCredentialSetError + if errors.As(buildErr, &malformed) { + a.r.loggers.Errorf("Malformed credential payload for offline environment %q — preserving previous credentials: %s", ae.Params.Identifiers.GetDisplayName(), buildErr) + } else { + // The environment was found above; this is a credential-build failure, not a missing env. + a.r.loggers.Errorf(logMsgAutoConfEnvInitError, ae.Params.Identifiers.GetDisplayName(), buildErr) } - env.UpdateCredential(update) + // Preserve previous credentials; no reconnect (offline path has no live stream). + } else { + env.ReconcileCredentials(set) } // SDKData will be non-nil only if the flag/segment data for the environment has actually changed. diff --git a/relay/filedata_actions_test.go b/relay/filedata_actions_test.go index 2540f7d9..a315fa53 100644 --- a/relay/filedata_actions_test.go +++ b/relay/filedata_actions_test.go @@ -231,7 +231,8 @@ func TestOfflineModeDeprecatedSDKKeyIsRespectedIfExpiryInFuture(t *testing.T) { env := p.awaitEnvironment(testFileDataEnv1.Params.EnvID) - assert.ElementsMatch(t, []credential.SDKCredential{envData.Params.SDKKey, envData.Params.EnvID}, env.GetCredentials()) + // Expiring key is in the accepted set (and thus GetCredentials) until it expires. + assert.ElementsMatch(t, []credential.SDKCredential{envData.Params.SDKKey, envData.Params.ExpiringSDKKey.Key, envData.Params.EnvID}, env.GetCredentials()) assert.ElementsMatch(t, []credential.SDKCredential{envData.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) }) } @@ -253,7 +254,8 @@ func TestOfflineModePrimarySDKKeyIsDeprecated(t *testing.T) { update2 := RotateSDKKeyWithGracePeriod("key2", "key1", time.Now().Add(1*time.Hour)) p.updateHandler.UpdateEnvironment(update2) - assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.SDKKey, update1.Params.EnvID}, env.GetCredentials()) + // Both the new anchor and the expiring old key are accepted until key1 expires. + assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.SDKKey, update2.Params.ExpiringSDKKey.Key, update1.Params.EnvID}, env.GetCredentials()) assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) update3 := RotateSDKKey("key3") @@ -261,9 +263,9 @@ func TestOfflineModePrimarySDKKeyIsDeprecated(t *testing.T) { assert.ElementsMatch(t, []credential.SDKCredential{update3.Params.SDKKey, update1.Params.EnvID}, env.GetCredentials()) - // Note: key2 isn't in the deprecated list, because update3 was an immediate rotation (with no grace period for the - // previous key.) At the same time, key1 is still deprecated until the hour is up. - assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) + // update3 carries no expiring key, so ReconcileCredentials replaces the full accepted set with + // just key3; key1 and key2 are removed immediately (not held in the deprecated bucket). + assert.Empty(t, env.GetDeprecatedCredentials()) }) } @@ -298,7 +300,8 @@ func TestOfflineModeSDKKeyCanExpire(t *testing.T) { // Waiting for the environment can take up to 1 second, but it could be much faster. In any case // we'll still need to sleep at least the cleanup interval to ensure the key is expired. env := p.awaitEnvironmentFor(update1.Params.EnvID, time.Second) - assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.EnvID}, env.GetCredentials()) + // Both the primary and the expiring key are in the accepted set until the expiry fires. + assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.ExpiringSDKKey.Key, update1.Params.EnvID}, env.GetCredentials()) assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) assert.Eventually(t, func() bool { diff --git a/relay/filedata_testdata_test.go b/relay/filedata_testdata_test.go index d097a4ce..7101e20e 100644 --- a/relay/filedata_testdata_test.go +++ b/relay/filedata_testdata_test.go @@ -66,6 +66,12 @@ func RotateSDKKey(primary config.SDKKey) filedata.ArchiveEnvironment { } func RotateSDKKeyWithGracePeriod(primary config.SDKKey, expiring config.SDKKey, expiry time.Time) filedata.ArchiveEnvironment { + // Populate AcceptedSDKKeys — the field BuildAcceptedSet uses — mirroring what env_rep.go's + // synthesis path produces for old-format payloads that carry sdkKey.expiring. + acceptedKeys := []envfactory.AcceptedSDKKey{{Value: primary}} + if expiring.Defined() { + acceptedKeys = append(acceptedKeys, envfactory.AcceptedSDKKey{Value: expiring, Expiry: expiry}) + } return filedata.ArchiveEnvironment{ Params: envfactory.EnvironmentParams{ EnvID: "env1", @@ -74,6 +80,7 @@ func RotateSDKKeyWithGracePeriod(primary config.SDKKey, expiring config.SDKKey, Key: expiring, Expiration: expiry, }, + AcceptedSDKKeys: acceptedKeys, Identifiers: relayenv.EnvIdentifiers{ ProjName: "Project", ProjKey: "project", diff --git a/relay/testutils_test.go b/relay/testutils_test.go index cd89c55e..80ed2b6f 100644 --- a/relay/testutils_test.go +++ b/relay/testutils_test.go @@ -3,7 +3,6 @@ package relay import ( "context" "net/http" - "reflect" "testing" "time" @@ -132,13 +131,17 @@ func (h relayTestHelper) assertEndpointStatus( } func (h relayTestHelper) awaitCredentialsUpdated(env relayenv.EnvContext, expected envfactory.EnvironmentParams) { - expectedCredentials := credentialsAsSet(expected.EnvID, expected.MobileKey, expected.SDKKey) - // Poll until both env.GetCredentials() and relay's connection mappings reflect the new credentials. - // The two updates are not atomic: AddCredential runs before AddConnectionMapping, so there is a - // window where GetCredentials() shows the new key but getEnvironment() still returns an error. + // Poll until the new expected credentials are present in env.GetCredentials() and the relay's + // connection mappings reflect them. GetCredentials() may also contain additional expiring keys + // from a previous rotation — we only require the expected ones to be present (subset check). isReady := func() bool { - if !reflect.DeepEqual(credentialsAsSet(env.GetCredentials()...), expectedCredentials) { - return false + actual := credentialsAsSet(env.GetCredentials()...) + for _, cred := range []credential.SDKCredential{expected.EnvID, expected.MobileKey, expected.SDKKey} { + if cred != nil && cred.(interface{ Defined() bool }).Defined() { + if _, ok := actual[cred]; !ok { + return false + } + } } for _, cred := range []sdkauth.ScopedCredential{ sdkauth.New(expected.EnvID), @@ -157,8 +160,16 @@ func (h relayTestHelper) awaitCredentialsUpdated(env relayenv.EnvContext, expect } func assertEnvProps(t *testing.T, expected envfactory.EnvironmentParams, env relayenv.EnvContext) { - assert.Equal(t, credentialsAsSet(expected.EnvID, expected.MobileKey, expected.SDKKey), - credentialsAsSet(env.GetCredentials()...)) + t.Helper() + // In the multi-key world GetCredentials() may contain additional expiring keys beyond the three + // "canonical" ones (anchor SDK key, primary mobile key, env ID). Check that the canonical ones are + // present rather than requiring exact equality. + actual := credentialsAsSet(env.GetCredentials()...) + for _, cred := range []credential.SDKCredential{expected.EnvID, expected.MobileKey, expected.SDKKey} { + if cred != nil && cred.(interface{ Defined() bool }).Defined() { + assert.Contains(t, actual, cred) + } + } assert.Equal(t, expected.Identifiers, env.GetIdentifiers()) assert.Equal(t, expected.Identifiers.ProjName+" "+expected.Identifiers.EnvName, env.GetIdentifiers().GetDisplayName()) From e44881fa59a3570325992778b18822d990681e32 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Sun, 28 Jun 2026 13:08:49 -0700 Subject: [PATCH 22/66] docs(concurrent-keys): correct T4 status-array spec to match implementation The status sdkKeys/mobileKeys arrays carry the full accepted set including the anchor/primary (per the backend tech spec and RAC payload spec), not a non-anchor subset. Drop the 'anchor first, then identifier-alphabetical' ordering requirement: it was a derived-doc embellishment with no basis in the source docs or Confluence (which treat every key as equally valid), so order is unspecified. Clarify the array-presence guarantee: always present, sdkKeys always >= 1 (the anchor), mobileKeys may be empty for a server-only env. --- .agent-docs/concurrent-keys/phase1-plan.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.agent-docs/concurrent-keys/phase1-plan.md b/.agent-docs/concurrent-keys/phase1-plan.md index 256c56c0..1197b754 100644 --- a/.agent-docs/concurrent-keys/phase1-plan.md +++ b/.agent-docs/concurrent-keys/phase1-plan.md @@ -288,11 +288,11 @@ Test matrix (covered in T3.c's acceptance criteria): ### T4 — Status endpoint arrays -Add `sdkKeys` / `mobileKeys` array fields to the env status response. Each entry: non-secret `Key` identifier + obscured `Value` (via `sdks.ObscureKey`) + optional `Expiry`. Keep scalar `sdkKey` / `mobileKey` — they now represent the **anchor** specifically. Keep `expiringSdkKey` for default-rotation back-compat. +Add `sdkKeys` / `mobileKeys` array fields to the env status response. The arrays carry the **full accepted set, including the anchor / primary mobile key** (mirroring the wire format). Each entry: optional non-secret `Key` identifier + obscured `Value` (via `sdks.ObscureKey`) + optional `Expiry`. Keep scalar `sdkKey` / `mobileKey` — they now designate which array entry is the **anchor** / primary, and stay for back-compat. Keep `expiringSdkKey` (the soonest-expiring non-anchor SDK key) for default-rotation back-compat. -Stable ordering of array entries: anchor first, then identifier-alphabetical. Predictable for tooling consumers. +Array entry order is unspecified — consumers look up by `key`/`value`, not position, and the spec treats every key as equally valid. (Stable ordering was a derived-doc embellishment, not a sourced requirement; tests are order-insensitive.) -Arrays are *present but empty* (not omitted) for single-key envs. +Arrays are always present (never omitted/null). `sdkKeys` always contains at least the anchor; `mobileKeys` may be empty for an environment with no mobile key (e.g. server-side only). ### T5.a — Integration test harness @@ -448,7 +448,7 @@ These scenarios live in code (as integration tests). This list is the *registry* | # | Scenario | Owner | |---|---|---| -| 20 | Status endpoint: scalar fields = anchor (obscured); arrays = full accepted set; per-key `expiry` visible when present; entries stably ordered (anchor first, identifier-alphabetical). | T4 | +| 20 | Status endpoint: scalar fields = anchor (obscured); arrays = full accepted set incl. anchor/primary; per-key `expiry` visible when present; arrays always present (`sdkKeys` ≥ 1, `mobileKeys` may be empty); order unspecified. | T4 | | 21 | Analytics events forwarded under the env's anchor key per kind, regardless of which accepted key the request came in on. | T2.c, T5.b | | 22 | Diagnostic events proxy verbatim under the originating credential (deliberate asymmetry — preserved, not collapsed). | T2.c | From 8e66e7934ced91751b72f51cc5f89a2044e2f09d Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Mon, 29 Jun 2026 08:28:22 -0700 Subject: [PATCH 23/66] =?UTF-8?q?refactor(credential):=20rename=20sdkKey?= =?UTF-8?q?=E2=86=92anchor=20(#722)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renames the anchor role (the single SDK key that owns the upstream connection) so it is not confused with “primary” when many SDK keys can be accepted at once. --- internal/credential/accepted_set.go | 6 +-- internal/credential/accepted_set_builder.go | 12 +++--- .../credential/accepted_set_builder_test.go | 10 ++--- internal/credential/rotator.go | 38 +++++++++--------- internal/credential/rotator_test.go | 40 +++++++++---------- internal/envfactory/reconcile_helper.go | 6 +-- internal/envfactory/reconcile_helper_test.go | 22 +++++----- internal/relayenv/env_context.go | 6 +-- internal/relayenv/env_context_impl.go | 8 ++-- internal/relayenv/env_context_impl_test.go | 22 +++++----- .../relayenv/env_context_reanchor_test.go | 6 +-- relay/endpoints_status.go | 2 +- 12 files changed, 89 insertions(+), 89 deletions(-) diff --git a/internal/credential/accepted_set.go b/internal/credential/accepted_set.go index 8de639b1..201bd9e6 100644 --- a/internal/credential/accepted_set.go +++ b/internal/credential/accepted_set.go @@ -13,11 +13,11 @@ import ( // expiry — plus the single environment ID and two primary designations: // // - The anchor: the one SDK key that owns the environment's upstream connection. Set with -// WithPrimarySDKKey. +// WithAnchor. // - The primary mobile key: the singular default mobile key (the wire's mobKey), used where one // mobile key is required, e.g. event forwarding. Set with WithPrimaryMobileKey. // -// WithPrimarySDKKey / WithPrimaryMobileKey both add the key to the set and designate it, so adding a +// WithAnchor / WithPrimaryMobileKey both add the key to the set and designate it, so adding a // single key takes one call. Build requires that an anchor was designated. (Structural validation of // the wire payload — undefined credentials, an anchor absent from the array — happens upstream when // the payload is parsed into the set; see SDK-2547.) @@ -31,7 +31,7 @@ type AcceptedSet struct { // without a containment scan. The map value is the key's expiry: a nil *time.Time means the key // is permanent. A nil map is a valid empty set (reads return absent; only the builder writes). sdkKeys map[config.SDKKey]*time.Time - primarySdkKey config.SDKKey + anchor config.SDKKey mobileKeys map[config.MobileKey]*time.Time primaryMobileKey config.MobileKey envID config.EnvironmentID diff --git a/internal/credential/accepted_set_builder.go b/internal/credential/accepted_set_builder.go index 7a3f0968..427f2943 100644 --- a/internal/credential/accepted_set_builder.go +++ b/internal/credential/accepted_set_builder.go @@ -36,12 +36,12 @@ func (b *AcceptedSetBuilder) WithExpiringSDKKey(key config.SDKKey, expiry time.T return b } -// WithPrimarySDKKey adds key (if not already present) and designates it as the anchor — the SDK key +// WithAnchor adds key (if not already present) and designates it as the anchor — the SDK key // that owns the environment's upstream connection. It is a no-op if the key is undefined. -func (b *AcceptedSetBuilder) WithPrimarySDKKey(key config.SDKKey) *AcceptedSetBuilder { +func (b *AcceptedSetBuilder) WithAnchor(key config.SDKKey) *AcceptedSetBuilder { if key.Defined() { b.addSDKKey(key, nil) - b.set.primarySdkKey = key + b.set.anchor = key } return b } @@ -99,13 +99,13 @@ func (b *AcceptedSetBuilder) WithEnvironmentID(id config.EnvironmentID) *Accepte // Build validates and returns the accumulated AcceptedSet. It returns errAcceptedSetMissingSDKKey if // no SDK key was added, or a *MalformedCredentialSetError if no anchor was designated (via -// WithPrimarySDKKey). Because WithPrimarySDKKey also adds the key, a designated anchor is always -// among the accepted SDK keys. +// WithAnchor). Because WithAnchor also adds the key, a designated anchor is always among the +// accepted SDK keys. func (b *AcceptedSetBuilder) Build() (AcceptedSet, error) { if len(b.set.sdkKeys) == 0 { return AcceptedSet{}, errAcceptedSetMissingSDKKey } - if !b.set.primarySdkKey.Defined() { + if !b.set.anchor.Defined() { return AcceptedSet{}, newMissingAnchorError() } return b.set, nil diff --git a/internal/credential/accepted_set_builder_test.go b/internal/credential/accepted_set_builder_test.go index b6e92d91..4ef837d1 100644 --- a/internal/credential/accepted_set_builder_test.go +++ b/internal/credential/accepted_set_builder_test.go @@ -22,25 +22,25 @@ func TestAcceptedSetBuilderValidation(t *testing.T) { _, err = NewAcceptedSetBuilder().WithSDKKey(config.SDKKey("sdk")).Build() require.ErrorAs(t, err, &malformed) - // WithPrimarySDKKey adds the key and designates it as the anchor, so Build succeeds. - set, err := NewAcceptedSetBuilder().WithPrimarySDKKey(config.SDKKey("sdk")).Build() + // WithAnchor adds the key and designates it as the anchor, so Build succeeds. + set, err := NewAcceptedSetBuilder().WithAnchor(config.SDKKey("sdk")).Build() require.NoError(t, err) assert.True(t, set.hasSDKKey(config.SDKKey("sdk"))) - assert.Equal(t, config.SDKKey("sdk"), set.primarySdkKey) + assert.Equal(t, config.SDKKey("sdk"), set.anchor) } func TestAcceptedSetBuilderDeduplicates(t *testing.T) { // Adding the same key more than once (including via WithPrimary*) keeps a single entry. set := mustBuild(t, NewAcceptedSetBuilder(). WithSDKKey(config.SDKKey("sdk")). - WithPrimarySDKKey(config.SDKKey("sdk")). + WithAnchor(config.SDKKey("sdk")). WithSDKKey(config.SDKKey("sdk")). WithMobileKey(config.MobileKey("mob")). WithPrimaryMobileKey(config.MobileKey("mob"))) assert.Len(t, set.sdkKeys, 1) assert.Len(t, set.mobileKeys, 1) - assert.Equal(t, config.SDKKey("sdk"), set.primarySdkKey) + assert.Equal(t, config.SDKKey("sdk"), set.anchor) assert.Equal(t, config.MobileKey("mob"), set.primaryMobileKey) } diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index c495282b..d57b3d7c 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -28,8 +28,8 @@ type Rotator struct { // here to allow setting it in a deferred manner. primaryEnvironmentID config.EnvironmentID - // There can be multiple SDK keys active at a given time, but only one is primary. - primarySdkKey config.SDKKey + // There can be multiple SDK keys active at a given time, but only one is the anchor. + anchorKey config.SDKKey // Deprecated keys are stored in a map with a started timer for each key representing the deprecation period. // Upon expiration, they are removed. @@ -76,7 +76,7 @@ func (r *Rotator) Initialize(credentials []SDKCredential) { } switch cred := cred.(type) { case config.SDKKey: - r.primarySdkKey = cred + r.anchorKey = cred r.acceptedSDKKeys[cred] = &acceptedKeyInfo{} case config.MobileKey: r.primaryMobileKey = cred @@ -94,11 +94,11 @@ func (r *Rotator) MobileKey() config.MobileKey { return r.primaryMobileKey } -// SDKKey returns the primary SDK key. -func (r *Rotator) SDKKey() config.SDKKey { +// AnchorKey returns the anchor SDK key — the key that owns the upstream connection. +func (r *Rotator) AnchorKey() config.SDKKey { r.mu.RLock() defer r.mu.RUnlock() - return r.primarySdkKey + return r.anchorKey } // EnvironmentID returns the environment ID. @@ -170,7 +170,7 @@ func (r *Rotator) DeprecatedCredentials() []SDKCredential { out := r.deprecatedCredentials() for key, info := range r.acceptedSDKKeys { - if info.expiry != nil && key != r.primarySdkKey { + if info.expiry != nil && key != r.anchorKey { out = append(out, key) } } @@ -285,22 +285,22 @@ func (r *Rotator) updateMobileKey(mobileKey config.MobileKey, grace *GracePeriod previous.Masked(), grace.expiry, mobileKey.Masked()) } -func (r *Rotator) swapPrimaryKey(newKey config.SDKKey) config.SDKKey { - if newKey == r.primarySdkKey { - // There's no swap to be done, we already are using this as primary. +func (r *Rotator) swapAnchor(newKey config.SDKKey) config.SDKKey { + if newKey == r.anchorKey { + // There's no swap to be done, we already are using this as the anchor. return "" } - previous := r.primarySdkKey - r.primarySdkKey = newKey + previous := r.anchorKey + r.anchorKey = newKey // Keep the accepted-set map (the source of truth for PrimaryCredentials) consistent: the new - // primary is accepted and is no longer deprecated, even if it was being phased out before. Mirrors + // anchor is accepted and is no longer deprecated, even if it was being phased out before. Mirrors // updateMobileKey for mobile keys. if _, ok := r.acceptedSDKKeys[newKey]; !ok { r.acceptedSDKKeys[newKey] = &acceptedKeyInfo{} } delete(r.deprecatedSdkKeys, newKey) r.additions = append(r.additions, newKey) - r.loggers.Infof("New primary SDK key is %s", newKey.Masked()) + r.loggers.Infof("New anchor SDK key is %s", newKey.Masked()) return previous } @@ -317,8 +317,8 @@ func (r *Rotator) updateSDKKey(sdkKey config.SDKKey, grace *GracePeriod) { r.mu.Lock() defer r.mu.Unlock() - // Previous will only be .Defined() if there was a previous primary key. - previous := r.swapPrimaryKey(sdkKey) + // Previous will only be .Defined() if there was a previous anchor key. + previous := r.swapAnchor(sdkKey) // If there's no deprecation notice, then the previous key (if any) needs to be immediately revoked so it doesn't // hang around forever. This case is also true when there is a grace period, but we need to inspect the grace period @@ -427,13 +427,13 @@ func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expiration // rather than reconcile. // // The set is assumed well-formed: AcceptedSetBuilder.Build validates that an anchor was designated -// (and, because WithPrimarySDKKey adds the key as it designates it, that the anchor is among the SDK +// (and, because WithAnchor adds the key as it designates it, that the anchor is among the SDK // keys), so Reconcile trusts what it is handed rather than re-validating. func (r *Rotator) Reconcile(set AcceptedSet, now time.Time) { r.mu.Lock() defer r.mu.Unlock() - r.reconcileSDKKeys(set, set.primarySdkKey, now) + r.reconcileSDKKeys(set, set.anchor, now) r.reconcileMobileKeys(set, now) r.reconcileEnvironmentID(set) } @@ -500,7 +500,7 @@ func (r *Rotator) reconcileSDKKeys(set AcceptedSet, anchor config.SDKKey, now ti } desired[anchor] = nil reconcileAcceptedKeys(desired, r.acceptedSDKKeys, r.deprecatedSdkKeys, &r.additions, &r.expirations, r.loggers, "SDK key") - r.primarySdkKey = anchor + r.anchorKey = anchor } // reconcileMobileKeys mirrors reconcileSDKKeys for mobile keys. The primary mobile key — the wire's diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index b5e7a67e..f1054df9 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -30,7 +30,7 @@ func TestImmediateKeyExpiration(t *testing.T) { { name: "sdk keys", keys: []SDKCredential{config.SDKKey("key1"), config.SDKKey("key2"), config.SDKKey("key3")}, - getKey: func(r *Rotator) SDKCredential { return r.SDKKey() }, + getKey: func(r *Rotator) SDKCredential { return r.AnchorKey() }, }, { name: "mobile keys", @@ -82,7 +82,7 @@ func TestManyImmediateKeyExpirations(t *testing.T) { { name: "sdk keys", makeKey: func(s string) SDKCredential { return config.SDKKey(s) }, - getKey: func(r *Rotator) SDKCredential { return r.SDKKey() }, + getKey: func(r *Rotator) SDKCredential { return r.AnchorKey() }, }, { name: "mobile keys", @@ -208,7 +208,7 @@ func TestManyConcurrentSDKKeyDeprecation(t *testing.T) { } // The last key added should be the current primary key. - assert.Equal(t, keysAdded[len(keysAdded)-1], rotator.SDKKey()) + assert.Equal(t, keysAdded[len(keysAdded)-1], rotator.AnchorKey()) // Until and including the exact expiry timestamp, there should be no expirations. additions, expirations := rotator.StepTime(expiryTime) @@ -254,7 +254,7 @@ func TestSDKKeyDeprecationWithAlreadyExpiredGraceRevokesPreviousPrimary(t *testi // accepted-but-untracked key. (This mirrors the equivalent mobile-key behavior.) rotator.RotateWithGrace(key2, NewGracePeriod(key1, expiry, now)) - assert.Equal(t, key2, rotator.SDKKey()) + assert.Equal(t, key2, rotator.AnchorKey()) additions, expirations := rotator.StepTime(now) assert.ElementsMatch(t, []SDKCredential{key2}, additions) @@ -282,7 +282,7 @@ func TestReAnchoringDeprecatedSDKKeyRemovesItFromDeprecatedSet(t *testing.T) { // Re-anchor key2 -> key1 before key1's grace expires. key1 must be promoted out of the // deprecated set; otherwise the cleanup ticker would later expire the active primary. rotator.Rotate(key1) - assert.Equal(t, key1, rotator.SDKKey()) + assert.Equal(t, key1, rotator.AnchorKey()) assert.Empty(t, rotator.DeprecatedCredentials()) additions, expirations := rotator.StepTime(start) @@ -293,7 +293,7 @@ func TestReAnchoringDeprecatedSDKKeyRemovesItFromDeprecatedSet(t *testing.T) { additions, expirations = rotator.StepTime(expiry.Add(1 * time.Hour)) assert.Empty(t, additions) assert.Empty(t, expirations) - assert.Equal(t, key1, rotator.SDKKey()) + assert.Equal(t, key1, rotator.AnchorKey()) } func TestInitializePopulatesAcceptedSets(t *testing.T) { @@ -319,7 +319,7 @@ func TestInitializePopulatesAcceptedSets(t *testing.T) { } // Existing public API is unchanged. - assert.Equal(t, sdkKey, rotator.SDKKey()) + assert.Equal(t, sdkKey, rotator.AnchorKey()) assert.Equal(t, mobileKey, rotator.MobileKey()) assert.Equal(t, envID, rotator.EnvironmentID()) } @@ -446,7 +446,7 @@ func TestRotateSDKKeyRePromoteClearsDeprecation(t *testing.T) { rotator.RotateWithGrace(key1, nil) // re-promote key1 rotator.StepTime(start) - assert.Equal(t, key1, rotator.SDKKey()) + assert.Equal(t, key1, rotator.AnchorKey()) assert.Contains(t, rotator.PrimaryCredentials(), SDKCredential(key1)) assert.NotContains(t, rotator.DeprecatedCredentials(), SDKCredential(key1)) } @@ -463,7 +463,7 @@ func TestRotateSDKKeyWithExpiredGraceRevokesPrevious(t *testing.T) { rotator.Initialize([]SDKCredential{key1}) rotator.RotateWithGrace(key2, NewGracePeriod(key1, expiry, now)) - assert.Equal(t, key2, rotator.SDKKey()) + assert.Equal(t, key2, rotator.AnchorKey()) additions, expirations := rotator.StepTime(now) assert.ElementsMatch(t, []SDKCredential{key2}, additions) assert.ElementsMatch(t, []SDKCredential{key1}, expirations) @@ -475,12 +475,12 @@ func TestReconcileAnchorOnly(t *testing.T) { anchor := config.SDKKey("anchor") now := time.Now() - r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor)), now) + r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor)), now) additions, expirations := r.StepTime(now) assert.ElementsMatch(t, []SDKCredential{anchor}, additions) assert.Empty(t, expirations) - assert.Equal(t, anchor, r.SDKKey()) + assert.Equal(t, anchor, r.AnchorKey()) assert.ElementsMatch(t, []SDKCredential{anchor}, r.PrimaryCredentials()) assert.Empty(t, r.DeprecatedCredentials()) } @@ -492,13 +492,13 @@ func TestReconcileMultipleSDKKeys(t *testing.T) { now := time.Now() r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor).WithSDKKey(other)), now) + mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor).WithSDKKey(other)), now) additions, expirations := r.StepTime(now) // Both server keys are accepted; only the anchor is primary. assert.ElementsMatch(t, []SDKCredential{anchor, other}, additions) assert.Empty(t, expirations) - assert.Equal(t, anchor, r.SDKKey()) + assert.Equal(t, anchor, r.AnchorKey()) assert.ElementsMatch(t, []SDKCredential{anchor, other}, r.PrimaryCredentials()) assert.Empty(t, r.DeprecatedCredentials()) } @@ -511,7 +511,7 @@ func TestReconcileMultipleMobileKeys(t *testing.T) { now := time.Now() r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor).WithPrimaryMobileKey(mob1).WithMobileKey(mob2)), now) + mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor).WithPrimaryMobileKey(mob1).WithMobileKey(mob2)), now) additions, _ := r.StepTime(now) // Every mobile key is accepted; the designated one is the primary. @@ -528,11 +528,11 @@ func TestReconcileRevokesOmittedKeys(t *testing.T) { now := time.Now() r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor).WithSDKKey(other).WithPrimaryMobileKey(mob)), now) + mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor).WithSDKKey(other).WithPrimaryMobileKey(mob)), now) r.StepTime(now) // Reconciling to just the anchor revokes the omitted server and mobile keys. - r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor)), now) + r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor)), now) additions, expirations := r.StepTime(now) assert.Empty(t, additions) @@ -554,7 +554,7 @@ func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { r.Reconcile( mustBuild(t, NewAcceptedSetBuilder(). - WithPrimarySDKKey(anchor). + WithAnchor(anchor). WithExpiringSDKKey(expiringSDK, now.Add(time.Hour)). WithPrimaryMobileKey(mob). WithExpiringMobileKey(expiringMobile, now.Add(time.Hour))), @@ -580,7 +580,7 @@ func TestReconcilePrimaryMobileKeyIsAlwaysAccepted(t *testing.T) { now := time.Unix(1000, 0) set := mustBuild(t, NewAcceptedSetBuilder(). - WithPrimarySDKKey(anchor). + WithAnchor(anchor). WithExpiringMobileKey(mob, now.Add(-time.Hour)). // already expired in the payload... WithPrimaryMobileKey(mob)) // ...but designated as the primary r.Reconcile(set, now) @@ -607,7 +607,7 @@ func TestReconcileClearsStaleDeprecationForAcceptedKey(t *testing.T) { // Reconcile to a set that fully accepts both keys. r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder().WithPrimarySDKKey(anchor).WithSDKKey(old)), now) + mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor).WithSDKKey(old)), now) r.StepTime(now) assert.Contains(t, r.PrimaryCredentials(), SDKCredential(old)) @@ -629,7 +629,7 @@ func TestReconcileExpiringKeysAreEvictedByStepTime(t *testing.T) { r.Reconcile( mustBuild(t, NewAcceptedSetBuilder(). - WithPrimarySDKKey(anchor). + WithAnchor(anchor). WithExpiringSDKKey(expiringSDK, expiry). WithPrimaryMobileKey(mob). WithExpiringMobileKey(expiringMobile, expiry)), diff --git a/internal/envfactory/reconcile_helper.go b/internal/envfactory/reconcile_helper.go index efa94ca0..2b262fdb 100644 --- a/internal/envfactory/reconcile_helper.go +++ b/internal/envfactory/reconcile_helper.go @@ -28,12 +28,12 @@ import ( func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config.SDKKey, error) { anchor := params.SDKKey - // WithPrimarySDKKey / WithPrimaryMobileKey each add the key and designate it (the anchor and the + // WithAnchor / WithPrimaryMobileKey each add the key and designate it (the anchor and the // wire's mobKey, respectively). An undefined key makes the call a no-op, so an undefined anchor // leaves the set with no designated anchor and Build returns a *MalformedCredentialSetError. b := credential.NewAcceptedSetBuilder(). WithEnvironmentID(params.EnvID). - WithPrimarySDKKey(anchor). + WithAnchor(anchor). WithPrimaryMobileKey(params.MobileKey) // Validate and add the remaining accepted keys. The builder de-duplicates by value, so the @@ -60,7 +60,7 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. // The anchor must be one of the accepted SDK keys: the backend lists it in sdkKeys[] (and ToParams // synthesizes it into the array for old-format payloads). A defined anchor absent from the array is - // a structurally malformed payload — reject it per §9 rather than letting WithPrimarySDKKey above + // a structurally malformed payload — reject it per §9 rather than letting WithAnchor above // silently synthesize it into the set. if anchor.Defined() && !anchorInArray { return credential.AcceptedSet{}, anchor, credential.NewAnchorNotInSetError() diff --git a/internal/envfactory/reconcile_helper_test.go b/internal/envfactory/reconcile_helper_test.go index 0f9b7711..2ab9a82e 100644 --- a/internal/envfactory/reconcile_helper_test.go +++ b/internal/envfactory/reconcile_helper_test.go @@ -56,7 +56,7 @@ func TestBuildAcceptedSet_HappyPath(t *testing.T) { expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithPrimarySDKKey("sdk-anchor"). + WithAnchor("sdk-anchor"). WithPrimaryMobileKey("mob-primary")) assert.Equal(t, expected, set) } @@ -80,7 +80,7 @@ func TestBuildAcceptedSet_MultipleKeys(t *testing.T) { expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithPrimarySDKKey("sdk-anchor"). + WithAnchor("sdk-anchor"). WithSDKKey("sdk-service-a"). WithExpiringSDKKey("sdk-old", expiry1). WithPrimaryMobileKey("mob-primary")) @@ -143,7 +143,7 @@ func TestBuildAcceptedSet_Deexpiry(t *testing.T) { // The set built without expiry must include sdk-old as a permanent key. expectedPermanent := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithPrimarySDKKey("sdk-anchor"). + WithAnchor("sdk-anchor"). WithSDKKey("sdk-old"). // permanent, no expiry WithPrimaryMobileKey("mob-primary")) assert.Equal(t, expectedPermanent, setNoExpiry) @@ -153,7 +153,7 @@ func TestBuildAcceptedSet_Deexpiry(t *testing.T) { } // TestBuildAcceptedSet_AnchorNotInArray verifies that an anchor absent from AcceptedSDKKeys is no -// longer rejected here: WithPrimarySDKKey adds and designates the anchor regardless, so the +// longer rejected here: WithAnchor adds and designates the anchor regardless, so the // resulting set contains both the anchor and the array entry. Structural validation of the wire // payload (anchor-absent-from-array) happens upstream when the payload is parsed into params. // TestBuildAcceptedSet_AnchorNotInArray verifies that a defined anchor absent from the sdkKeys[] array @@ -192,7 +192,7 @@ func TestBuildAcceptedSet_NoMobileKey(t *testing.T) { expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithPrimarySDKKey("sdk-anchor")) + WithAnchor("sdk-anchor")) assert.Equal(t, expected, set) } @@ -252,7 +252,7 @@ func TestBuildAcceptedSet_MixedUpdate(t *testing.T) { expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithPrimarySDKKey("sdk-new-anchor"). + WithAnchor("sdk-new-anchor"). WithSDKKey("sdk-b"). WithSDKKey("sdk-c"). WithPrimaryMobileKey("mob-primary")) @@ -276,10 +276,10 @@ func TestBuildAcceptedSet_AnchorNeverExpiring(t *testing.T) { require.NoError(t, err) assert.Equal(t, config.SDKKey("sdk-anchor"), anchor) - // Anchor is permanent (WithPrimarySDKKey), not expiring — identical to a payload with no anchor expiry. + // Anchor is permanent (WithAnchor), not expiring — identical to a payload with no anchor expiry. expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithPrimarySDKKey("sdk-anchor"). + WithAnchor("sdk-anchor"). WithSDKKey("sdk-service-a"). WithPrimaryMobileKey("mob-primary")) assert.Equal(t, expected, set) @@ -304,7 +304,7 @@ func TestBuildAcceptedSet_MultipleMobileKeys(t *testing.T) { require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithPrimarySDKKey("sdk-anchor"). + WithAnchor("sdk-anchor"). WithMobileKey("mob-primary"). WithMobileKey("mob-secondary"). WithPrimaryMobileKey("mob-primary")) @@ -331,7 +331,7 @@ func TestBuildAcceptedSet_ExpiringMobileKey(t *testing.T) { require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithPrimarySDKKey("sdk-anchor"). + WithAnchor("sdk-anchor"). WithMobileKey("mob-primary"). WithExpiringMobileKey("mob-old", expiry1). WithPrimaryMobileKey("mob-primary")) @@ -361,7 +361,7 @@ func TestBuildAcceptedSet_TrustTheArray(t *testing.T) { require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithPrimarySDKKey("sdk-anchor"). + WithAnchor("sdk-anchor"). WithPrimaryMobileKey("mob-primary")) assert.Equal(t, expected, set, "legacy sdkKey.expiring slot must not appear in AcceptedSet") } diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index 61073b19..123ae722 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -40,12 +40,12 @@ type EnvContext interface { // SetIdentifiers updates the environment and project names and keys. SetIdentifiers(EnvIdentifiers) - // GetSDKKey returns the anchor SDK key — the primary key that owns the upstream connection. + // GetAnchorKey returns the anchor SDK key — the key that owns the upstream connection. // Use this when you need exactly the anchor (e.g. the status endpoint's sdkKey field) rather // than the full accepted set returned by GetCredentials. - GetSDKKey() config.SDKKey + GetAnchorKey() config.SDKKey - // GetMobileKey returns the primary (default) mobile key. Like GetSDKKey for the anchor, use this + // GetMobileKey returns the primary (default) mobile key. Like GetAnchorKey for the anchor, use this // for the status endpoint's mobileKey field rather than iterating GetCredentials, which may return // several accepted mobile keys (primary + expiring) in nondeterministic order. GetMobileKey() config.MobileKey diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 7f9516e6..ded8d3a6 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -451,7 +451,7 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { // So, the effect in offline mode when adding/removing credentials is just setting up the new credential mappings. switch key := newCredential.(type) { case config.SDKKey: - if key == c.keyRotator.SDKKey() { + if key == c.keyRotator.AnchorKey() { if !c.offline { go c.startSDKClient(key, nil, false) } @@ -628,8 +628,8 @@ func (c *envContextImpl) GetCredentials() []credential.SDKCredential { return c.keyRotator.PrimaryCredentials() } -func (c *envContextImpl) GetSDKKey() config.SDKKey { - return c.keyRotator.SDKKey() +func (c *envContextImpl) GetAnchorKey() config.SDKKey { + return c.keyRotator.AnchorKey() } func (c *envContextImpl) GetMobileKey() config.MobileKey { @@ -656,7 +656,7 @@ func (c *envContextImpl) GetClient() sdks.LDClientContext { } return nil } - return c.clients[c.keyRotator.SDKKey()] + return c.clients[c.keyRotator.AnchorKey()] } func (c *envContextImpl) GetStore() subsystems.DataStore { diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 3368cfa3..2f1cc44e 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -203,7 +203,7 @@ func TestAddRemoveCredential(t *testing.T) { // Reconcile to the full set: the SDK key (anchor) plus a mobile key and an environment ID. env.ReconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithPrimarySDKKey(envConfig.SDKKey).WithPrimaryMobileKey(mobileKey).WithEnvironmentID(envID))) + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(envConfig.SDKKey).WithPrimaryMobileKey(mobileKey).WithEnvironmentID(envID))) creds := env.GetCredentials() assert.Len(t, creds, 3) @@ -214,7 +214,7 @@ func TestAddRemoveCredential(t *testing.T) { // Reconciling with a different mobile key evicts the previous one. newMobileKey := config.MobileKey("evict-the-previous-key") env.ReconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithPrimarySDKKey(envConfig.SDKKey).WithPrimaryMobileKey(newMobileKey).WithEnvironmentID(envID))) + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(envConfig.SDKKey).WithPrimaryMobileKey(newMobileKey).WithEnvironmentID(envID))) creds = env.GetCredentials() assert.Len(t, creds, 3) @@ -236,7 +236,7 @@ func TestAddExistingCredentialDoesNothing(t *testing.T) { assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) mobileKey := st.EnvWithAllCredentials.Config.MobileKey - set := mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithPrimarySDKKey(envConfig.SDKKey).WithPrimaryMobileKey(mobileKey)) + set := mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(envConfig.SDKKey).WithPrimaryMobileKey(mobileKey)) env.ReconcileCredentials(set) @@ -286,7 +286,7 @@ func TestChangeSDKKey(t *testing.T) { // Upon rotating to key2, the original key should still be valid for an hour. rotationSet, err := credential.NewAcceptedSetBuilder(). - WithPrimarySDKKey(key2). + WithAnchor(key2). WithExpiringSDKKey(envConfig.SDKKey, start.Add(1*time.Hour)). Build() require.NoError(t, err) @@ -397,7 +397,7 @@ func TestMobileKeyReconcileExpiry(t *testing.T) { // carries a per-key expiry. envImpl.reconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithPrimarySDKKey(envConfig.SDKKey). + WithAnchor(envConfig.SDKKey). WithPrimaryMobileKey(primaryMobile). WithExpiringMobileKey(expiringMobile, expiry)), start) @@ -442,7 +442,7 @@ func TestNonAnchorSDKKeysDoNotOpenUpstreamClient(t *testing.T) { // open an upstream client. env.ReconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithPrimarySDKKey(envConfig.SDKKey). + WithAnchor(envConfig.SDKKey). WithSDKKey(nonAnchorKey1). WithSDKKey(nonAnchorKey2))) @@ -487,7 +487,7 @@ func TestGetClientReturnsAnchorInMultiKeyEnv(t *testing.T) { env.ReconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithPrimarySDKKey(envConfig.SDKKey). + WithAnchor(envConfig.SDKKey). WithSDKKey(nonAnchorKey1). WithSDKKey(nonAnchorKey2))) @@ -530,7 +530,7 @@ func TestNonPrimaryMobileKeyDoesNotStealEventForwarding(t *testing.T) { // non-primary mobile key. Accepting the non-primary key must NOT repoint event forwarding — // events collapse to the primary mobile key, mirroring the SDK anchor. env.ReconcileCredentials(mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithPrimarySDKKey(envConfig.SDKKey). + WithAnchor(envConfig.SDKKey). WithPrimaryMobileKey(primaryMobile). WithMobileKey(nonPrimaryMobile). WithEnvironmentID(envConfig.EnvID))) @@ -590,7 +590,7 @@ func TestReAnchoringToKeyStillInGraceReusesItsClient(t *testing.T) { // alive because keyA is still accepted during the grace window. env.(*envContextImpl).reconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithPrimarySDKKey(keyB). + WithAnchor(keyB). WithExpiringSDKKey(keyA, start.Add(1*time.Hour))), start) @@ -605,7 +605,7 @@ func TestReAnchoringToKeyStillInGraceReusesItsClient(t *testing.T) { // being started -- so there is no stale client to orphan. keyB is omitted from the set (no expiry), // so it is revoked immediately and its client is closed. env.(*envContextImpl).reconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithPrimarySDKKey(keyA)), + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(keyA)), start.Add(10*time.Minute)) // keyB was revoked by the re-anchor, so its client is closed. @@ -680,7 +680,7 @@ func TestRevokingSDKKeyWhileClientIsStartingDoesNotLeakTheClient(t *testing.T) { // runs now -- but c.clients[keyA] is still nil because the initial goroutine is blocked in the factory, // so nothing is closed and the mapping is simply removed. env.(*envContextImpl).reconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithPrimarySDKKey(keyB)), + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(keyB)), time.Unix(1000, 0)) creds := env.GetCredentials() diff --git a/internal/relayenv/env_context_reanchor_test.go b/internal/relayenv/env_context_reanchor_test.go index ff05b693..ee6e8c2a 100644 --- a/internal/relayenv/env_context_reanchor_test.go +++ b/internal/relayenv/env_context_reanchor_test.go @@ -103,7 +103,7 @@ func (f *sharedStoreFactory) Build(_ subsystems.ClientContext) (subsystems.DataS func reanchor(t *testing.T, env EnvContext, newKey, oldKey config.SDKKey, now time.Time) { t.Helper() set, err := credential.NewAcceptedSetBuilder(). - WithPrimarySDKKey(newKey). + WithAnchor(newKey). WithExpiringSDKKey(oldKey, now.Add(time.Hour)). Build() require.NoError(t, err) @@ -562,8 +562,8 @@ func TestReanchorPoC_H6_AnchorPointerFlipsBeforeNewClientIsRegistered(t *testing reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) // FINDING: there is a window where the anchor pointer already names the new key but no client exists - // for it yet, so GetClient() returns nil. GetClient() == clients[rotator.SDKKey()], and the rotator's - // primary flipped to the new key before startSDKClient registered the client. A request arriving in + // for it yet, so GetClient() returns nil. GetClient() == clients[rotator.AnchorKey()], and the rotator's + // anchor flipped to the new key before startSDKClient registered the client. A request arriving in // this window gets a nil client. T2.c must not advance the anchor pointer until the new client is // registered (and ideally Initialized()). assert.Nil(t, env.GetClient(), "GetClient() is nil during the swap window") diff --git a/relay/endpoints_status.go b/relay/endpoints_status.go index 01546a7b..cce75869 100644 --- a/relay/endpoints_status.go +++ b/relay/endpoints_status.go @@ -49,7 +49,7 @@ func statusHandler(relay *Relay) http.Handler { // Use the anchor SDK key and primary mobile key specifically — GetCredentials() may return // multiple SDK and mobile keys (primary + expiring), so iterating it for these singular // status fields would give a non-deterministic result. - if key := clientCtx.GetSDKKey(); key.Defined() { + if key := clientCtx.GetAnchorKey(); key.Defined() { status.SDKKey = sdks.ObscureKey(string(key)) } if key := clientCtx.GetMobileKey(); key.Defined() { From 991b11b2bb65d5b6ec9f86d6ff860ec76d4d9f38 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Mon, 29 Jun 2026 08:47:35 -0700 Subject: [PATCH 24/66] refactor(credential): remove legacy single-key rotation path from Rotator (#723) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the legacy imperative credential rotation API (Rotate, RotateWithGrace, GracePeriod, and deprecatedSdkKeys / deprecatedMobileKeys) from internal/credential/rotator.go. Credential state is now maintained only through Reconcile and the acceptedSDKKeys / acceptedMobileKeys maps, where “deprecated” keys are simply accepted entries with a non-nil expiry. --- internal/credential/accepted_set_test.go | 5 - internal/credential/rotator.go | 294 +--------- internal/credential/rotator_test.go | 622 ++------------------- internal/relayenv/env_context_impl.go | 2 +- internal/relayenv/env_context_impl_test.go | 39 -- relay/filedata_actions_test.go | 43 +- 6 files changed, 107 insertions(+), 898 deletions(-) diff --git a/internal/credential/accepted_set_test.go b/internal/credential/accepted_set_test.go index 4519b1ee..f8f3c58f 100644 --- a/internal/credential/accepted_set_test.go +++ b/internal/credential/accepted_set_test.go @@ -3,8 +3,6 @@ package credential import ( "testing" - "github.com/launchdarkly/ld-relay/v8/config" - "github.com/stretchr/testify/assert" ) @@ -15,7 +13,4 @@ func TestMalformedCredentialSetErrorMessage(t *testing.T) { // Empty credential value. assert.Contains(t, NewEmptyCredentialError("sdkKeys", "my-key").Error(), "empty value") - - // The config.SDKKey import is exercised; confirm it still compiles. - _ = config.SDKKey("sdk-abcd1234") } diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index d57b3d7c..728305ad 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -19,11 +19,6 @@ type Rotator struct { // There is only one mobile key active at a given time. primaryMobileKey config.MobileKey - // deprecatedMobileKeys stores mobile keys being phased out with a grace period, keyed - // by credential value with the associated expiry time. StepTime walks this map and emits - // an expiration once a key's grace period passes, mirroring deprecatedSdkKeys. - deprecatedMobileKeys map[config.MobileKey]time.Time - // There is only one environment ID active at a given time, and it won't actually be rotated. The mechanism is // here to allow setting it in a deferred manner. primaryEnvironmentID config.EnvironmentID @@ -31,12 +26,12 @@ type Rotator struct { // There can be multiple SDK keys active at a given time, but only one is the anchor. anchorKey config.SDKKey - // Deprecated keys are stored in a map with a started timer for each key representing the deprecation period. - // Upon expiration, they are removed. - deprecatedSdkKeys map[config.SDKKey]time.Time + // acceptedSDKKeys is the full set of accepted SDK keys with optional per-key expiry. + // A nil expiry means the key is permanent. The anchor is always present with a nil expiry. + acceptedSDKKeys map[config.SDKKey]*acceptedKeyInfo - // Consumed by ReconcileCredentials API - acceptedSDKKeys map[config.SDKKey]*acceptedKeyInfo + // acceptedMobileKeys is the full set of accepted mobile keys with optional per-key expiry. + // A nil expiry means the key is permanent. acceptedMobileKeys map[config.MobileKey]*acceptedKeyInfo expirations []SDKCredential @@ -55,11 +50,9 @@ type InitialCredentials struct { // contains no credentials and can optionally be initialized via Initialize. func NewRotator(loggers ldlog.Loggers) *Rotator { r := &Rotator{ - loggers: loggers, - deprecatedSdkKeys: make(map[config.SDKKey]time.Time), - deprecatedMobileKeys: make(map[config.MobileKey]time.Time), - acceptedSDKKeys: make(map[config.SDKKey]*acceptedKeyInfo), - acceptedMobileKeys: make(map[config.MobileKey]*acceptedKeyInfo), + loggers: loggers, + acceptedSDKKeys: make(map[config.SDKKey]*acceptedKeyInfo), + acceptedMobileKeys: make(map[config.MobileKey]*acceptedKeyInfo), } return r } @@ -108,30 +101,14 @@ func (r *Rotator) EnvironmentID() config.EnvironmentID { return r.primaryEnvironmentID } -// PrimaryCredentials returns the primary (non-deprecated) credentials. -func (r *Rotator) PrimaryCredentials() []SDKCredential { - r.mu.RLock() - defer r.mu.RUnlock() - return r.primaryCredentials() -} - -// primaryCredentials returns every accepted, non-deprecated credential: all accepted SDK keys, all -// accepted mobile keys, and the environment ID. The primary SDK key and primary mobile key are always -// present in the accepted-set maps (maintained by Initialize, the legacy rotation path, and Reconcile) -// and are never left marked deprecated, so a plain pass over the maps already includes them. -func (r *Rotator) primaryCredentials() []SDKCredential { +// allCredentials returns every accepted credential. Expiring keys are included until the +// cleanup ticker drops them (StepTime). The caller must hold at least a read lock. +func (r *Rotator) allCredentials() []SDKCredential { creds := make([]SDKCredential, 0, len(r.acceptedSDKKeys)+len(r.acceptedMobileKeys)+1) - for key := range r.acceptedSDKKeys { - if _, deprecated := r.deprecatedSdkKeys[key]; deprecated { - continue - } creds = append(creds, key) } for key := range r.acceptedMobileKeys { - if _, deprecated := r.deprecatedMobileKeys[key]; deprecated { - continue - } creds = append(creds, key) } if r.primaryEnvironmentID.Defined() { @@ -140,17 +117,6 @@ func (r *Rotator) primaryCredentials() []SDKCredential { return creds } -func (r *Rotator) deprecatedCredentials() []SDKCredential { - deprecated := make([]SDKCredential, 0, len(r.deprecatedSdkKeys)+len(r.deprecatedMobileKeys)) - for key := range r.deprecatedSdkKeys { - deprecated = append(deprecated, key) - } - for key := range r.deprecatedMobileKeys { - deprecated = append(deprecated, key) - } - return deprecated -} - // DeprecatedCredentials returns the SDK keys being phased out — every accepted SDK key, other than the // anchor, that carries a future expiry. (Per-key expiry is stored as data on the accepted entry; the // cleanup ticker drops the key once it elapses.) EnvContext.GetDeprecatedCredentials delegates here to @@ -163,12 +129,7 @@ func (r *Rotator) DeprecatedCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() - // TEMPORARY (legacy rotation path): keys deprecated via RotateWithGrace live in the - // deprecatedSdkKeys / deprecatedMobileKeys buckets, which the reconcile path never populates. Once - // the legacy path is removed (SDK-2603) these buckets are always empty; delete this line and the - // deprecatedCredentials helper, leaving only the accepted-with-expiry logic below. - out := r.deprecatedCredentials() - + var out []SDKCredential for key, info := range r.acceptedSDKKeys { if info.expiry != nil && key != r.anchorKey { out = append(out, key) @@ -177,230 +138,41 @@ func (r *Rotator) DeprecatedCredentials() []SDKCredential { return out } -// AllCredentials returns the primary and deprecated credentials as one list. +// AllCredentials returns every accepted credential: every accepted SDK key, every accepted mobile +// key (including those carrying a future expiry — they still authenticate until the cleanup ticker +// drops them), and the environment ID. func (r *Rotator) AllCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() - return append(r.primaryCredentials(), r.deprecatedCredentials()...) -} - -// Rotate sets a new primary credential while revoking the previous. -func (r *Rotator) Rotate(cred SDKCredential) { - r.RotateWithGrace(cred, nil) -} - -// GracePeriod represents a grace period (or deprecation period) within which -// a particular SDK key is still valid, pending revocation. -type GracePeriod struct { - // The SDK key that is being deprecated. - key config.SDKKey - // When the key will expire. - expiry time.Time - // The current timestamp. - now time.Time -} - -// Expired returns true if the key has already expired. -func (g *GracePeriod) Expired() bool { - return g.now.After(g.expiry) -} - -// NewGracePeriod constructs a new grace period. The current time must be provided in order to -// determine if the credential is already expired. -func NewGracePeriod(key config.SDKKey, expiry time.Time, now time.Time) *GracePeriod { - return &GracePeriod{key, expiry, now} -} - -// RotateWithGrace sets a new primary credential while deprecating the previous one. When grace is nil -// the outgoing credential is immediately revoked. It is invalid to specify a grace period for an -// environment ID. For mobile keys, a non-nil grace period stores the expiry for the outgoing key; -// the cleanup ticker is responsible for acting on it. -func (r *Rotator) RotateWithGrace(primary SDKCredential, grace *GracePeriod) { - switch primary := primary.(type) { - case config.SDKKey: - r.updateSDKKey(primary, grace) - case config.MobileKey: - r.updateMobileKey(primary, grace) - case config.EnvironmentID: - if grace != nil { - panic("programmer error: environment IDs do not support deprecation") - } - r.updateEnvironmentID(primary) - } -} - -func (r *Rotator) updateEnvironmentID(envID config.EnvironmentID) { - if envID == r.EnvironmentID() { - return - } - r.mu.Lock() - defer r.mu.Unlock() - previous := r.primaryEnvironmentID - r.primaryEnvironmentID = envID - r.additions = append(r.additions, envID) - if previous.Defined() { - r.loggers.Infof("Environment ID %s was rotated, new environment ID is %s", r.primaryEnvironmentID, envID) - r.expirations = append(r.expirations, previous) - } else { - r.loggers.Infof("New environment ID is %s", envID) - } -} - -// updateMobileKey sets a new primary mobile key. When grace is nil the outgoing key is -// immediately revoked; when non-nil its expiry is stored in deprecatedMobileKeys for the -// cleanup ticker (StepTime) to act on. -func (r *Rotator) updateMobileKey(mobileKey config.MobileKey, grace *GracePeriod) { - r.mu.Lock() - defer r.mu.Unlock() - if mobileKey == r.primaryMobileKey { - return - } - previous := r.primaryMobileKey - r.primaryMobileKey = mobileKey - // Keep the accepted-set map (the source of truth for PrimaryCredentials) consistent with the - // legacy rotation path. - if _, ok := r.acceptedMobileKeys[mobileKey]; !ok { - r.acceptedMobileKeys[mobileKey] = &acceptedKeyInfo{} - } - delete(r.deprecatedMobileKeys, mobileKey) - r.additions = append(r.additions, mobileKey) - if !previous.Defined() { - r.loggers.Infof("New primary mobile key is %s", mobileKey.Masked()) - return - } - if grace == nil { - delete(r.acceptedMobileKeys, previous) - r.expirations = append(r.expirations, previous) - r.loggers.Infof("Mobile key %s was rotated, new primary mobile key is %s", previous.Masked(), mobileKey.Masked()) - return - } - if grace.Expired() { - delete(r.acceptedMobileKeys, previous) - r.loggers.Infof("Deprecated mobile key %s already expired at %v; revoking immediately", previous.Masked(), grace.expiry) - r.expirations = append(r.expirations, previous) - return - } - r.deprecatedMobileKeys[previous] = grace.expiry - r.loggers.Infof("Mobile key %s was marked for deprecation with an expiry at %v, new primary mobile key is %s", - previous.Masked(), grace.expiry, mobileKey.Masked()) -} - -func (r *Rotator) swapAnchor(newKey config.SDKKey) config.SDKKey { - if newKey == r.anchorKey { - // There's no swap to be done, we already are using this as the anchor. - return "" - } - previous := r.anchorKey - r.anchorKey = newKey - // Keep the accepted-set map (the source of truth for PrimaryCredentials) consistent: the new - // anchor is accepted and is no longer deprecated, even if it was being phased out before. Mirrors - // updateMobileKey for mobile keys. - if _, ok := r.acceptedSDKKeys[newKey]; !ok { - r.acceptedSDKKeys[newKey] = &acceptedKeyInfo{} - } - delete(r.deprecatedSdkKeys, newKey) - r.additions = append(r.additions, newKey) - r.loggers.Infof("New anchor SDK key is %s", newKey.Masked()) - - return previous -} - -func (r *Rotator) immediatelyRevoke(key config.SDKKey) { - if key.Defined() { - delete(r.acceptedSDKKeys, key) - r.expirations = append(r.expirations, key) - r.loggers.Infof("SDK key %s has been immediately revoked", key.Masked()) - } -} - -func (r *Rotator) updateSDKKey(sdkKey config.SDKKey, grace *GracePeriod) { - r.mu.Lock() - defer r.mu.Unlock() - - // Previous will only be .Defined() if there was a previous anchor key. - previous := r.swapAnchor(sdkKey) - - // If there's no deprecation notice, then the previous key (if any) needs to be immediately revoked so it doesn't - // hang around forever. This case is also true when there is a grace period, but we need to inspect the grace period - // in order to find out if immediate revocation is necessary. - if grace == nil { - r.immediatelyRevoke(previous) - return - } - - if previousExpiry, ok := r.deprecatedSdkKeys[grace.key]; ok { - if previousExpiry != grace.expiry { - r.loggers.Warnf("SDK key %s was marked for deprecation with an expiry at %v, but it was previously deprecated with an expiry at %v. The previous expiry will be used. ", grace.key.Masked(), grace.expiry, previousExpiry) - } - // When a key is deprecated by LD, it will stick around in the deprecated field of the message until something - // else is deprecated. This means that if a key is rotated *without* a deprecation period set for the previous key, - // then we'll receive that new primary key but the deprecation message will be stale - it'll be referring to the - // last time a key was rotated with a deprecation period. We detect this case here (since we already saw the - // deprecation message in our map) and ensure the previous key is revoked. - r.immediatelyRevoke(previous) - return - } - - if grace.Expired() { - r.loggers.Infof("Deprecated SDK key %s already expired at %v; revoking the previous key immediately", grace.key.Masked(), grace.expiry) - r.immediatelyRevoke(previous) - return - } - - r.loggers.Infof("SDK key %s was marked for deprecation with an expiry at %v", grace.key.Masked(), grace.expiry) - r.deprecatedSdkKeys[grace.key] = grace.expiry - - if grace.key != previous { - r.loggers.Infof("Deprecated SDK key %s was not previously managed by Relay", grace.key.Masked()) - r.additions = append(r.additions, grace.key) - } + return r.allCredentials() } func (r *Rotator) expireSDKKey(sdkKey config.SDKKey) { r.loggers.Infof("Deprecated SDK key %s has expired and is no longer valid for authentication", sdkKey.Masked()) - delete(r.deprecatedSdkKeys, sdkKey) delete(r.acceptedSDKKeys, sdkKey) r.expirations = append(r.expirations, sdkKey) } -// expireMobileKey drops a mobile key from both the deprecated grace map and the accepted set, then -// queues its expiration. Deleting from acceptedMobileKeys is load-bearing: PrimaryCredentials derives -// from that map, so an expired key would otherwise linger as a primary credential. Mirrors expireSDKKey. +// expireMobileKey drops a mobile key from the accepted set and queues its expiration. +// Deleting from acceptedMobileKeys is load-bearing: AllCredentials derives from that map, +// so an expired key would otherwise linger as an accepted credential. Mirrors expireSDKKey. func (r *Rotator) expireMobileKey(mobileKey config.MobileKey) { r.loggers.Infof("Deprecated mobile key %s has expired and is no longer valid for authentication", mobileKey.Masked()) - delete(r.deprecatedMobileKeys, mobileKey) delete(r.acceptedMobileKeys, mobileKey) r.expirations = append(r.expirations, mobileKey) } -// StepTime provides the current time to the Rotator, allowing it to compute the set of additions and expirations -// for the tracked credentials since the last time this method was called. +// StepTime provides the current time to the Rotator, allowing it to compute the set of additions and +// expirations for the tracked credentials since the last time this method was called. // -// It enforces expiry from both expiry mechanisms, for both SDK and mobile keys: -// - The legacy grace-period maps (deprecatedSdkKeys / deprecatedMobileKeys), populated by the -// RotateWithGrace path, where the expiry lives in the map value. -// - The reconcile path, where per-key expiry is stored as data on the accepted entry -// (acceptedKeyInfo.expiry); a nil expiry means the key is permanent and is never expired here. +// It enforces per-key expiry for both SDK and mobile keys: expiry is stored as data on the accepted +// entry (acceptedKeyInfo.expiry); a nil expiry means the key is permanent and is never expired here. // -// Expiry is strict (now strictly after the expiry timestamp), consistent across all four loops. +// Expiry happens strictly after a key's expiry timestamp. func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expirations []SDKCredential) { r.mu.Lock() defer r.mu.Unlock() - // Legacy grace-period deprecations (RotateWithGrace path). - for key, expiry := range r.deprecatedSdkKeys { - if now.After(expiry) { - r.expireSDKKey(key) - } - } - for key, expiry := range r.deprecatedMobileKeys { - if now.After(expiry) { - r.expireMobileKey(key) - } - } - - // Reconcile-path per-key expiry, stored on the accepted entry. The anchor and primary mobile key - // carry a nil expiry, so this never drops them. for key, info := range r.acceptedSDKKeys { if info.expiry != nil && now.After(*info.expiry) { r.expireSDKKey(key) @@ -447,22 +219,18 @@ type reconcilableKey interface { // reconcileAcceptedKeys diffs the desired keys against the currently-accepted ones (SDK or mobile, // same algorithm): a desired key not yet accepted is recorded and queued as an addition; an accepted -// key no longer desired is dropped and queued as an expiration. Either way the key is cleared from the -// deprecated map — a key the set accepts is not deprecated, and a key it revokes is gone. Per-key -// expiry is stored as data on the accepted entry; the cleanup ticker is what later acts on it. The -// caller must hold the write lock. +// key no longer desired is dropped and queued as an expiration. Per-key expiry is stored as data on +// the accepted entry; the cleanup ticker is what later acts on it. The caller must hold the write lock. func reconcileAcceptedKeys[K reconcilableKey]( desired map[K]*time.Time, accepted map[K]*acceptedKeyInfo, - deprecated map[K]time.Time, additions *[]SDKCredential, expirations *[]SDKCredential, loggers ldlog.Loggers, kind string, ) { // First pass: walk every key the set wants us to accept. If we already accept it, just refresh - // its expiry; if it's new, start accepting it and queue it as an addition. Either way, a desired - // key can't also be deprecated, so clear any stale deprecation for it. + // its expiry; if it's new, start accepting it and queue it as an addition. for key, expiry := range desired { if info, ok := accepted[key]; ok { info.expiry = expiry @@ -471,17 +239,15 @@ func reconcileAcceptedKeys[K reconcilableKey]( *additions = append(*additions, key) loggers.Infof("%s %s is now accepted", kind, key.Masked()) } - delete(deprecated, key) } // Second pass: walk every key we currently accept and drop the ones the set no longer wants. // Keys still desired were handled above, so skip them; the rest are revoked outright (removed - // from both maps) and queued as expirations. + // from the map) and queued as expirations. for key := range accepted { if _, ok := desired[key]; ok { continue } delete(accepted, key) - delete(deprecated, key) *expirations = append(*expirations, key) loggers.Infof("%s %s is no longer accepted and has been revoked", kind, key.Masked()) } @@ -499,7 +265,7 @@ func (r *Rotator) reconcileSDKKeys(set AcceptedSet, anchor config.SDKKey, now ti desired[key] = expiry } desired[anchor] = nil - reconcileAcceptedKeys(desired, r.acceptedSDKKeys, r.deprecatedSdkKeys, &r.additions, &r.expirations, r.loggers, "SDK key") + reconcileAcceptedKeys(desired, r.acceptedSDKKeys, &r.additions, &r.expirations, r.loggers, "SDK key") r.anchorKey = anchor } @@ -517,7 +283,7 @@ func (r *Rotator) reconcileMobileKeys(set AcceptedSet, now time.Time) { if set.primaryMobileKey.Defined() { desired[set.primaryMobileKey] = nil } - reconcileAcceptedKeys(desired, r.acceptedMobileKeys, r.deprecatedMobileKeys, &r.additions, &r.expirations, r.loggers, "Mobile key") + reconcileAcceptedKeys(desired, r.acceptedMobileKeys, &r.additions, &r.expirations, r.loggers, "Mobile key") r.primaryMobileKey = set.primaryMobileKey } diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index f1054df9..b682c8f2 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -1,7 +1,6 @@ package credential import ( - "fmt" "testing" "time" @@ -21,281 +20,6 @@ func TestNewRotator(t *testing.T) { assert.NotNil(t, rotator) } -func TestImmediateKeyExpiration(t *testing.T) { - kinds := []struct { - name string - keys []SDKCredential - getKey func(*Rotator) SDKCredential - }{ - { - name: "sdk keys", - keys: []SDKCredential{config.SDKKey("key1"), config.SDKKey("key2"), config.SDKKey("key3")}, - getKey: func(r *Rotator) SDKCredential { return r.AnchorKey() }, - }, - { - name: "mobile keys", - keys: []SDKCredential{config.MobileKey("key1"), config.MobileKey("key2"), config.MobileKey("key3")}, - getKey: func(r *Rotator) SDKCredential { return r.MobileKey() }, - }, - { - name: "environment IDs", - keys: []SDKCredential{config.EnvironmentID("id1"), config.EnvironmentID("id2"), config.EnvironmentID("id3")}, - getKey: func(r *Rotator) SDKCredential { return r.EnvironmentID() }, - }, - } - - for _, c := range kinds { - t.Run(c.name, func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - // The first rotation shouldn't trigger any expirations because there was no previous key. - rotator.Rotate(c.keys[0]) - additions, _ := rotator.StepTime(time.Now()) - assert.ElementsMatch(t, c.keys[0:1], additions) - assert.Equal(t, c.keys[0], c.getKey(rotator)) - - // The second rotation should trigger a deprecation of key1. - rotator.Rotate(c.keys[1]) - additions, expirations := rotator.StepTime(time.Now()) - assert.ElementsMatch(t, c.keys[1:2], additions) - assert.ElementsMatch(t, c.keys[0:1], expirations) - assert.Equal(t, c.keys[1], c.getKey(rotator)) - - // The third rotation should trigger a deprecation of key2. - rotator.Rotate(c.keys[2]) - additions, expirations = rotator.StepTime(time.Now()) - assert.ElementsMatch(t, c.keys[2:3], additions) - assert.ElementsMatch(t, c.keys[1:2], expirations) - assert.Equal(t, c.keys[2], c.getKey(rotator)) - }) - } -} - -func TestManyImmediateKeyExpirations(t *testing.T) { - - kinds := []struct { - name string - makeKey func(string) SDKCredential - getKey func(*Rotator) SDKCredential - }{ - { - name: "sdk keys", - makeKey: func(s string) SDKCredential { return config.SDKKey(s) }, - getKey: func(r *Rotator) SDKCredential { return r.AnchorKey() }, - }, - { - name: "mobile keys", - makeKey: func(s string) SDKCredential { return config.MobileKey(s) }, - getKey: func(r *Rotator) SDKCredential { return r.MobileKey() }, - }, - { - name: "environment IDs", - makeKey: func(s string) SDKCredential { return config.EnvironmentID(s) }, - getKey: func(r *Rotator) SDKCredential { return r.EnvironmentID() }, - }, - } - - for _, c := range kinds { - t.Run(c.name, func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - const numKeys = 100 - for i := 0; i < numKeys; i++ { - key := c.makeKey(fmt.Sprintf("key%v", i)) - rotator.Rotate(key) - } - - assert.Equal(t, c.makeKey(fmt.Sprintf("key%v", numKeys-1)), c.getKey(rotator)) - - additions, expirations := rotator.StepTime(time.Now()) - assert.Len(t, additions, numKeys) - assert.Len(t, expirations, numKeys-1) // because the last key is still active - }) - } -} - -func TestImmediateSDKKeyDeprecationEvenIfGracePeriodIsPresent(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - key0 := config.SDKKey("key0") - key1 := config.SDKKey("key1") - key2 := config.SDKKey("key2") - - rotator.Initialize([]SDKCredential{key0}) - - start := time.Unix(1000, 0) - halftime := start.Add(30 * time.Minute) - expiry := start.Add(1 * time.Hour) - - rotator.RotateWithGrace(key1, NewGracePeriod(key0, expiry, start)) - - additions, expirations := rotator.StepTime(halftime) - assert.ElementsMatch(t, []SDKCredential{key1}, additions) - assert.Empty(t, expirations) - - // The deprecated key0 given here can be thought of as "stale" or otherwise already-seen by the rotator. - // In this case, it should be effectively ignored but the new key2 should still trigger rotation of the previous - // primary key. - rotator.RotateWithGrace(key2, NewGracePeriod(key0, expiry, halftime)) - - additions, expirations = rotator.StepTime(halftime) - assert.ElementsMatch(t, []SDKCredential{key2}, additions) - assert.ElementsMatch(t, []SDKCredential{key1}, expirations) - - additions, expirations = rotator.StepTime(expiry.Add(1 * time.Millisecond)) - assert.Empty(t, additions) - assert.ElementsMatch(t, []SDKCredential{key0}, expirations) -} - -func TestSDKKeyDeprecation(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - const ( - key1 = config.SDKKey("key1") - key2 = config.SDKKey("key2") - ) - - start := time.Unix(10000, 0) - - halfTime := start.Add(30 * time.Second) - deprecationTime := start.Add(1 * time.Minute) - - rotator.Initialize([]SDKCredential{key1}) - - rotator.RotateWithGrace(key2, NewGracePeriod(key1, deprecationTime, halfTime)) - additions, expirations := rotator.StepTime(halfTime) - assert.ElementsMatch(t, []SDKCredential{key2}, additions) - assert.Empty(t, expirations) - - additions, expirations = rotator.StepTime(deprecationTime) - assert.Empty(t, additions) - assert.Empty(t, expirations) - - additions, expirations = rotator.StepTime(deprecationTime.Add(1 * time.Millisecond)) - assert.Empty(t, additions) - assert.ElementsMatch(t, []SDKCredential{key1}, expirations) -} - -func TestManyConcurrentSDKKeyDeprecation(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - makeKey := func(i int) config.SDKKey { - return config.SDKKey(fmt.Sprintf("key%v", i)) - } - - rotator.Initialize([]SDKCredential{config.SDKKey("key0")}) - - const numKeys = 250 - now := time.Unix(10000, 0) - expiryTime := now.Add(1 * time.Hour) - - var keysDeprecated []SDKCredential - var keysAdded []SDKCredential - - for i := 0; i < numKeys; i++ { - previousKey := makeKey(i) - nextKey := makeKey(i + 1) - - keysDeprecated = append(keysDeprecated, previousKey) - keysAdded = append(keysAdded, nextKey) - - rotator.RotateWithGrace(nextKey, NewGracePeriod(previousKey, expiryTime, now)) - } - - // The last key added should be the current primary key. - assert.Equal(t, keysAdded[len(keysAdded)-1], rotator.AnchorKey()) - - // Until and including the exact expiry timestamp, there should be no expirations. - additions, expirations := rotator.StepTime(expiryTime) - assert.ElementsMatch(t, keysAdded, additions) - assert.Empty(t, expirations) - - // One moment after the expiry time, we should now have a batch of expirations. - additions, expirations = rotator.StepTime(expiryTime.Add(1 * time.Millisecond)) - assert.Empty(t, additions) - assert.ElementsMatch(t, keysDeprecated, expirations) -} - -func TestSDKKeyExpiredInThePastIsNotAdded(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - primaryKey := config.SDKKey("primary") - obsoleteKey := config.SDKKey("obsolete") - obsoleteExpiry := time.Unix(1000000, 0) - now := obsoleteExpiry.Add(1 * time.Hour) - - rotator.RotateWithGrace(primaryKey, NewGracePeriod(obsoleteKey, obsoleteExpiry, now)) - - additions, expirations := rotator.StepTime(now) - assert.ElementsMatch(t, []SDKCredential{primaryKey}, additions) - assert.Empty(t, expirations) -} - -func TestSDKKeyDeprecationWithAlreadyExpiredGraceRevokesPreviousPrimary(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - key1 := config.SDKKey("key1") - key2 := config.SDKKey("key2") - - expiry := time.Unix(10000, 0) - now := expiry.Add(1 * time.Hour) // now is after the grace period's expiry - - rotator.Initialize([]SDKCredential{key1}) - - // Rotate key1 -> key2, but the deprecation grace for the outgoing key1 has already elapsed. - // key2 becomes primary; key1 must be revoked immediately rather than lingering forever as an - // accepted-but-untracked key. (This mirrors the equivalent mobile-key behavior.) - rotator.RotateWithGrace(key2, NewGracePeriod(key1, expiry, now)) - - assert.Equal(t, key2, rotator.AnchorKey()) - - additions, expirations := rotator.StepTime(now) - assert.ElementsMatch(t, []SDKCredential{key2}, additions) - assert.ElementsMatch(t, []SDKCredential{key1}, expirations) - assert.Empty(t, rotator.DeprecatedCredentials()) -} - -func TestReAnchoringDeprecatedSDKKeyRemovesItFromDeprecatedSet(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - key1 := config.SDKKey("key1") - key2 := config.SDKKey("key2") - - start := time.Unix(10000, 0) - expiry := start.Add(1 * time.Hour) - - rotator.Initialize([]SDKCredential{key1}) - - // Rotate key1 -> key2 with grace; key1 enters the deprecated set. - rotator.RotateWithGrace(key2, NewGracePeriod(key1, expiry, start)) - rotator.StepTime(start) - assert.ElementsMatch(t, []SDKCredential{key1}, rotator.DeprecatedCredentials()) - - // Re-anchor key2 -> key1 before key1's grace expires. key1 must be promoted out of the - // deprecated set; otherwise the cleanup ticker would later expire the active primary. - rotator.Rotate(key1) - assert.Equal(t, key1, rotator.AnchorKey()) - assert.Empty(t, rotator.DeprecatedCredentials()) - - additions, expirations := rotator.StepTime(start) - assert.ElementsMatch(t, []SDKCredential{key1}, additions) - assert.ElementsMatch(t, []SDKCredential{key2}, expirations) - - // Well past the original grace expiry, key1 (the active primary) must NOT be expired. - additions, expirations = rotator.StepTime(expiry.Add(1 * time.Hour)) - assert.Empty(t, additions) - assert.Empty(t, expirations) - assert.Equal(t, key1, rotator.AnchorKey()) -} - func TestInitializePopulatesAcceptedSets(t *testing.T) { mockLog := ldlogtest.NewMockLog() rotator := NewRotator(mockLog.Loggers) @@ -324,152 +48,6 @@ func TestInitializePopulatesAcceptedSets(t *testing.T) { assert.Equal(t, envID, rotator.EnvironmentID()) } -func TestRotateWithGraceMobileKey(t *testing.T) { - t.Run("does not panic with non-nil grace period", func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - mob1 := config.MobileKey("mob1") - mob2 := config.MobileKey("mob2") - - start := time.Unix(10000, 0) - expiry := start.Add(1 * time.Hour) - - rotator.Initialize([]SDKCredential{mob1}) - - // GracePeriod.key is SDK-key typed; pass a zero value since mobile-key rotation - // does not use that identifier field. - assert.NotPanics(t, func() { - rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), expiry, start)) - }) - - assert.Equal(t, mob2, rotator.MobileKey()) - - // mob2 is a new addition; mob1 is in the deprecated set (not yet expired), - // so it should not appear as an expiration here. - additions, expirations := rotator.StepTime(start) - assert.ElementsMatch(t, []SDKCredential{mob2}, additions) - assert.Empty(t, expirations) - - // One moment past the grace period, the cleanup ticker expires mob1 and evicts it from the - // accepted set entirely. - additions, expirations = rotator.StepTime(expiry.Add(1 * time.Millisecond)) - assert.Empty(t, additions) - assert.ElementsMatch(t, []SDKCredential{mob1}, expirations) - assert.NotContains(t, rotator.PrimaryCredentials(), SDKCredential(mob1)) - assert.NotContains(t, rotator.DeprecatedCredentials(), SDKCredential(mob1)) - }) - - t.Run("immediately revokes outgoing key when grace period is already expired", func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - mob1 := config.MobileKey("mob1") - mob2 := config.MobileKey("mob2") - - expiry := time.Unix(10000, 0) - now := expiry.Add(1 * time.Hour) // now is after expiry - - rotator.Initialize([]SDKCredential{mob1}) - - rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), expiry, now)) - - assert.Equal(t, mob2, rotator.MobileKey()) - - additions, expirations := rotator.StepTime(now) - assert.ElementsMatch(t, []SDKCredential{mob2}, additions) - assert.ElementsMatch(t, []SDKCredential{mob1}, expirations) - // The immediately-revoked key must leave the accepted set, not linger in PrimaryCredentials. - assert.NotContains(t, rotator.PrimaryCredentials(), SDKCredential(mob1)) - }) - - t.Run("immediately revokes outgoing key when grace is nil", func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - mob1 := config.MobileKey("mob1") - mob2 := config.MobileKey("mob2") - - rotator.Initialize([]SDKCredential{mob1}) - rotator.RotateWithGrace(mob2, nil) - - assert.Equal(t, mob2, rotator.MobileKey()) - - additions, expirations := rotator.StepTime(time.Now()) - assert.ElementsMatch(t, []SDKCredential{mob2}, additions) - assert.ElementsMatch(t, []SDKCredential{mob1}, expirations) - // The immediately-revoked key must leave the accepted set, not linger in PrimaryCredentials. - assert.NotContains(t, rotator.PrimaryCredentials(), SDKCredential(mob1)) - }) - - t.Run("re-promoting a deprecated key removes it from the deprecated set", func(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - mob1 := config.MobileKey("mob1") - mob2 := config.MobileKey("mob2") - - start := time.Unix(10000, 0) - expiry := start.Add(1 * time.Hour) - - rotator.Initialize([]SDKCredential{mob1}) - - // Rotate mob1 → mob2 with grace; mob1 enters deprecatedMobileKeys. - rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), expiry, start)) - rotator.StepTime(start) - - // Rotate back mob2 → mob1; mob1 should be promoted out of the deprecated set. - rotator.RotateWithGrace(mob1, nil) - - assert.Equal(t, mob1, rotator.MobileKey()) - - // mob1 should appear only as an addition, not also as an expiration. - additions, expirations := rotator.StepTime(start) - assert.ElementsMatch(t, []SDKCredential{mob1}, additions) - assert.ElementsMatch(t, []SDKCredential{mob2}, expirations) - }) -} - -func TestRotateSDKKeyRePromoteClearsDeprecation(t *testing.T) { - // Re-promoting a deprecated SDK key back to primary must clear its deprecated mark, so - // PrimaryCredentials lists it (mirrors the mobile re-promote behavior). - rotator := newTestRotator() - key1 := config.SDKKey("key1") - key2 := config.SDKKey("key2") - start := time.Unix(10000, 0) - - rotator.Initialize([]SDKCredential{key1}) - rotator.RotateWithGrace(key2, NewGracePeriod(key1, start.Add(time.Hour), start)) // deprecate key1 - rotator.StepTime(start) - assert.ElementsMatch(t, []SDKCredential{key1}, rotator.DeprecatedCredentials()) - - rotator.RotateWithGrace(key1, nil) // re-promote key1 - rotator.StepTime(start) - - assert.Equal(t, key1, rotator.AnchorKey()) - assert.Contains(t, rotator.PrimaryCredentials(), SDKCredential(key1)) - assert.NotContains(t, rotator.DeprecatedCredentials(), SDKCredential(key1)) -} - -func TestRotateSDKKeyWithExpiredGraceRevokesPrevious(t *testing.T) { - // A legacy SDK rotation whose grace period is already expired must revoke the swapped-out key, - // not leave it enabled alongside the new anchor (mirrors updateMobileKey). - rotator := newTestRotator() - key1 := config.SDKKey("key1") - key2 := config.SDKKey("key2") - expiry := time.Unix(10000, 0) - now := expiry.Add(time.Hour) // now is after expiry - - rotator.Initialize([]SDKCredential{key1}) - rotator.RotateWithGrace(key2, NewGracePeriod(key1, expiry, now)) - - assert.Equal(t, key2, rotator.AnchorKey()) - additions, expirations := rotator.StepTime(now) - assert.ElementsMatch(t, []SDKCredential{key2}, additions) - assert.ElementsMatch(t, []SDKCredential{key1}, expirations) - assert.NotContains(t, rotator.PrimaryCredentials(), SDKCredential(key1)) -} - func TestReconcileAnchorOnly(t *testing.T) { r := newTestRotator() anchor := config.SDKKey("anchor") @@ -481,7 +59,7 @@ func TestReconcileAnchorOnly(t *testing.T) { assert.ElementsMatch(t, []SDKCredential{anchor}, additions) assert.Empty(t, expirations) assert.Equal(t, anchor, r.AnchorKey()) - assert.ElementsMatch(t, []SDKCredential{anchor}, r.PrimaryCredentials()) + assert.ElementsMatch(t, []SDKCredential{anchor}, r.AllCredentials()) assert.Empty(t, r.DeprecatedCredentials()) } @@ -499,7 +77,7 @@ func TestReconcileMultipleSDKKeys(t *testing.T) { assert.ElementsMatch(t, []SDKCredential{anchor, other}, additions) assert.Empty(t, expirations) assert.Equal(t, anchor, r.AnchorKey()) - assert.ElementsMatch(t, []SDKCredential{anchor, other}, r.PrimaryCredentials()) + assert.ElementsMatch(t, []SDKCredential{anchor, other}, r.AllCredentials()) assert.Empty(t, r.DeprecatedCredentials()) } @@ -517,7 +95,7 @@ func TestReconcileMultipleMobileKeys(t *testing.T) { // Every mobile key is accepted; the designated one is the primary. assert.ElementsMatch(t, []SDKCredential{anchor, mob1, mob2}, additions) assert.Equal(t, mob1, r.MobileKey()) - assert.ElementsMatch(t, []SDKCredential{anchor, mob1, mob2}, r.PrimaryCredentials()) + assert.ElementsMatch(t, []SDKCredential{anchor, mob1, mob2}, r.AllCredentials()) } func TestReconcileRevokesOmittedKeys(t *testing.T) { @@ -537,12 +115,12 @@ func TestReconcileRevokesOmittedKeys(t *testing.T) { assert.Empty(t, additions) assert.ElementsMatch(t, []SDKCredential{other, mob}, expirations) - assert.ElementsMatch(t, []SDKCredential{anchor}, r.PrimaryCredentials()) + assert.ElementsMatch(t, []SDKCredential{anchor}, r.AllCredentials()) } func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { // Reconcile stores per-key expiry as data on the accepted entry; before that expiry passes, an - // expiring key is still accepted (it authenticates and appears in PrimaryCredentials) while also + // expiring key is still accepted (it authenticates and appears in AllCredentials) while also // being reported as deprecated — accepted, but on its way out. The cleanup ticker (StepTime) only // drops it once the expiry elapses — see TestReconcileExpiringKeysAreEvictedByStepTime. r := newTestRotator() @@ -564,7 +142,7 @@ func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { assert.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, additions) assert.Empty(t, expirations) // Every key is accepted (still authenticates)... - assert.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, r.PrimaryCredentials()) + assert.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, r.AllCredentials()) // ...and the non-anchor SDK key carrying an expiry is also reported as deprecated (being phased // out). The expiring mobile key is not: there is no expiringMobileKey status field, so the reconcile // path treats it as accepted-only. @@ -573,7 +151,7 @@ func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { func TestReconcilePrimaryMobileKeyIsAlwaysAccepted(t *testing.T) { // Defensive: even if the designated primary mobile key is also listed with a past expiry, it must - // stay accepted (mirroring the SDK anchor), so PrimaryCredentials never reports a torn-down key. + // stay accepted (mirroring the SDK anchor), so AllCredentials never reports a torn-down key. r := newTestRotator() anchor := config.SDKKey("anchor") mob := config.MobileKey("mob") @@ -587,38 +165,15 @@ func TestReconcilePrimaryMobileKeyIsAlwaysAccepted(t *testing.T) { r.StepTime(now) assert.Equal(t, mob, r.MobileKey()) - assert.Contains(t, r.PrimaryCredentials(), SDKCredential(mob)) + assert.Contains(t, r.AllCredentials(), SDKCredential(mob)) _, accepted := r.acceptedMobileKeys[mob] assert.True(t, accepted, "the primary mobile key must remain in the accepted set") } -func TestReconcileClearsStaleDeprecationForAcceptedKey(t *testing.T) { - // A key left in the deprecated set by the legacy rotation path must be treated as fully accepted - // once a reconcile includes it, not silently skipped by PrimaryCredentials. - r := newTestRotator() - old := config.SDKKey("old") - anchor := config.SDKKey("anchor") - now := time.Unix(1000, 0) - - r.Initialize([]SDKCredential{old}) - r.RotateWithGrace(anchor, NewGracePeriod(old, now.Add(time.Hour), now)) // deprecate `old` with grace - r.StepTime(now) - require.ElementsMatch(t, []SDKCredential{old}, r.DeprecatedCredentials()) - - // Reconcile to a set that fully accepts both keys. - r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor).WithSDKKey(old)), now) - r.StepTime(now) - - assert.Contains(t, r.PrimaryCredentials(), SDKCredential(old)) - assert.Empty(t, r.DeprecatedCredentials()) -} - func TestReconcileExpiringKeysAreEvictedByStepTime(t *testing.T) { // End-to-end on the reconcile path: a reconcile records per-key expiry as data on the accepted // entry, and the cleanup ticker (StepTime) later drops both the expiring SDK key and the expiring - // mobile key once their expiry elapses — without ever passing through the legacy deprecated maps. - // The anchor and primary mobile key carry no expiry and survive. + // mobile key once their expiry elapses. The anchor and primary mobile key carry no expiry and survive. r := newTestRotator() anchor := config.SDKKey("anchor") expiringSDK := config.SDKKey("expiring-sdk") @@ -647,125 +202,66 @@ func TestReconcileExpiringKeysAreEvictedByStepTime(t *testing.T) { additions, expirations = r.StepTime(expiry.Add(1 * time.Millisecond)) assert.Empty(t, additions) assert.ElementsMatch(t, []SDKCredential{expiringSDK, expiringMobile}, expirations) - assert.ElementsMatch(t, []SDKCredential{anchor, mob}, r.PrimaryCredentials()) - assert.NotContains(t, r.PrimaryCredentials(), SDKCredential(expiringSDK)) - assert.NotContains(t, r.PrimaryCredentials(), SDKCredential(expiringMobile)) + assert.ElementsMatch(t, []SDKCredential{anchor, mob}, r.AllCredentials()) + assert.NotContains(t, r.AllCredentials(), SDKCredential(expiringSDK)) + assert.NotContains(t, r.AllCredentials(), SDKCredential(expiringMobile)) } -func TestMobileKeyDeprecation(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - const ( - mob1 = config.MobileKey("mob1") - mob2 = config.MobileKey("mob2") - ) - - start := time.Unix(10000, 0) - halfTime := start.Add(30 * time.Second) - deprecationTime := start.Add(1 * time.Minute) - - rotator.Initialize([]SDKCredential{mob1}) - - rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), deprecationTime, halfTime)) - additions, expirations := rotator.StepTime(halfTime) - assert.ElementsMatch(t, []SDKCredential{mob2}, additions) - assert.Empty(t, expirations) - - // At the exact expiry, not yet expired. - additions, expirations = rotator.StepTime(deprecationTime) - assert.Empty(t, additions) - assert.Empty(t, expirations) - - // One moment past the expiry: mob1 is expired. - additions, expirations = rotator.StepTime(deprecationTime.Add(1 * time.Millisecond)) - assert.Empty(t, additions) - assert.ElementsMatch(t, []SDKCredential{mob1}, expirations) -} - -func TestManyConcurrentMobileKeyDeprecation(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - makeKey := func(i int) config.MobileKey { - return config.MobileKey(fmt.Sprintf("mob%v", i)) - } - - rotator.Initialize([]SDKCredential{makeKey(0)}) - - const numKeys = 50 - now := time.Unix(10000, 0) - expiryTime := now.Add(1 * time.Hour) - - var keysDeprecated []SDKCredential - var keysAdded []SDKCredential - - for i := 0; i < numKeys; i++ { - nextKey := makeKey(i + 1) - keysDeprecated = append(keysDeprecated, makeKey(i)) - keysAdded = append(keysAdded, nextKey) - rotator.RotateWithGrace(nextKey, NewGracePeriod(config.SDKKey(""), expiryTime, now)) - } +func TestReconcileAlreadyExpiredKeyIsIgnoredOnAdd(t *testing.T) { + // An entry in the reconcile payload whose expiry is already in the past is treated as absent — + // the reconcile path filters it before calling reconcileAcceptedKeys, so it is never added. + r := newTestRotator() + anchor := config.SDKKey("anchor") + staleKey := config.SDKKey("stale") + now := time.Unix(2000, 0) + alreadyExpired := now.Add(-time.Hour) - assert.Equal(t, keysAdded[len(keysAdded)-1], rotator.MobileKey()) + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(anchor). + WithExpiringSDKKey(staleKey, alreadyExpired)), + now) + additions, expirations := r.StepTime(now) - // Until and including the exact expiry timestamp, no expirations. - additions, expirations := rotator.StepTime(expiryTime) - assert.ElementsMatch(t, keysAdded, additions) + // Only the anchor is added; the stale key is never accepted. + assert.ElementsMatch(t, []SDKCredential{anchor}, additions) assert.Empty(t, expirations) - - // One moment after the expiry time: batch of expirations. - additions, expirations = rotator.StepTime(expiryTime.Add(1 * time.Millisecond)) - assert.Empty(t, additions) - assert.ElementsMatch(t, keysDeprecated, expirations) + assert.ElementsMatch(t, []SDKCredential{anchor}, r.AllCredentials()) } -func TestMixedSDKAndMobileKeyExpiry(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - sdk1 := config.SDKKey("sdk1") - sdk2 := config.SDKKey("sdk2") - mob1 := config.MobileKey("mob1") - mob2 := config.MobileKey("mob2") - - now := time.Unix(10000, 0) - expiry := now.Add(1 * time.Hour) +func TestReconcileDeExpiryRestoresKey(t *testing.T) { + // When a key was accepted with a future expiry and a subsequent reconcile removes that expiry + // (de-expiry), the key becomes permanent: the cleanup ticker will no longer drop it, and it is + // no longer reported as deprecated. + r := newTestRotator() + anchor := config.SDKKey("anchor") + key := config.SDKKey("other") + now := time.Unix(1000, 0) + expiry := now.Add(time.Hour) - rotator.Initialize([]SDKCredential{sdk1, mob1}) + // First reconcile: key is accepted with a future expiry. + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(anchor). + WithExpiringSDKKey(key, expiry)), + now) + r.StepTime(now) + require.ElementsMatch(t, []SDKCredential{key}, r.DeprecatedCredentials()) - rotator.RotateWithGrace(sdk2, NewGracePeriod(sdk1, expiry, now)) - rotator.StepTime(now) + // Second reconcile: same key, no expiry (de-expiry). + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(anchor). + WithSDKKey(key)), + now) + r.StepTime(now) - rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), expiry, now)) - rotator.StepTime(now) + // The key is still accepted and permanent: no longer deprecated, not evicted by StepTime. + assert.Contains(t, r.AllCredentials(), SDKCredential(key)) + assert.Empty(t, r.DeprecatedCredentials()) - // Both sdk1 and mob1 should expire at the same tick. - additions, expirations := rotator.StepTime(expiry.Add(1 * time.Millisecond)) + additions, expirations := r.StepTime(expiry.Add(1 * time.Millisecond)) assert.Empty(t, additions) - assert.ElementsMatch(t, []SDKCredential{sdk1, mob1}, expirations) -} - -func TestDeprecatedCredentialsIncludesMobileKeys(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - - sdk1 := config.SDKKey("sdk1") - sdk2 := config.SDKKey("sdk2") - mob1 := config.MobileKey("mob1") - mob2 := config.MobileKey("mob2") - - now := time.Unix(10000, 0) - expiry := now.Add(1 * time.Hour) - - rotator.Initialize([]SDKCredential{sdk1, mob1}) - - rotator.RotateWithGrace(sdk2, NewGracePeriod(sdk1, expiry, now)) - rotator.StepTime(now) - - rotator.RotateWithGrace(mob2, NewGracePeriod(config.SDKKey(""), expiry, now)) - rotator.StepTime(now) - - deprecated := rotator.DeprecatedCredentials() - assert.ElementsMatch(t, []SDKCredential{sdk1, mob1}, deprecated) + assert.Empty(t, expirations) + assert.Contains(t, r.AllCredentials(), SDKCredential(key)) } diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index ded8d3a6..964d9659 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -625,7 +625,7 @@ func (c *envContextImpl) triggerCredentialChanges(now time.Time) { } func (c *envContextImpl) GetCredentials() []credential.SDKCredential { - return c.keyRotator.PrimaryCredentials() + return c.keyRotator.AllCredentials() } func (c *envContextImpl) GetAnchorKey() config.SDKKey { diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 2f1cc44e..654e8d9e 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -332,45 +332,6 @@ func TestChangeSDKKey(t *testing.T) { } -// TestMobileKeyGraceExpiry drives a deprecated mobile key (legacy RotateWithGrace path) to expiry -// through the cleanup ticker (triggerCredentialChanges → StepTime), mirroring the SDK-key flow in -// TestChangeSDKKey. -func TestMobileKeyGraceExpiry(t *testing.T) { - envConfig := st.EnvMobile.Config - readyCh := make(chan EnvContext, 1) - - mob1 := envConfig.MobileKey - mob2 := config.MobileKey("mob2-new-key") - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactory(true), mockLog.Loggers, readyCh) - defer env.Close() - envImpl := env.(*envContextImpl) - - assert.Equal(t, env, requireEnvReady(t, readyCh)) - - start := time.Unix(2000, 0) - graceDuration := 1 * time.Hour - - // Rotate mob1 → mob2 with a grace period; mob1 is deprecated but still valid. - envImpl.keyRotator.RotateWithGrace(mob2, credential.NewGracePeriod(config.SDKKey(""), start.Add(graceDuration), start)) - envImpl.triggerCredentialChanges(start) - - assert.Contains(t, env.GetDeprecatedCredentials(), mob1) - - // Before the grace period ends, mob1 is still deprecated (not yet expired). - envImpl.triggerCredentialChanges(start.Add(30 * time.Minute)) - assert.Contains(t, env.GetDeprecatedCredentials(), mob1) - - // One moment past the grace period: the cleanup ticker evicts mob1 entirely. - envImpl.triggerCredentialChanges(start.Add(graceDuration + 1*time.Millisecond)) - - assert.NotContains(t, env.GetCredentials(), mob1) - assert.NotContains(t, env.GetDeprecatedCredentials(), mob1) -} - // TestMobileKeyReconcileExpiry drives a mobile key carrying a per-key expiry end-to-end through the // reconcile path: ReconcileCredentials records the expiry as data on the accepted entry, and the // cleanup ticker (triggerCredentialChanges → StepTime) later evicts the key once its expiry elapses. diff --git a/relay/filedata_actions_test.go b/relay/filedata_actions_test.go index a315fa53..ace1f7de 100644 --- a/relay/filedata_actions_test.go +++ b/relay/filedata_actions_test.go @@ -1,7 +1,6 @@ package relay import ( - "fmt" "net/http" "net/http/httptest" "sort" @@ -35,8 +34,6 @@ import ( // calls the same Relay methods that the real ArchiveManager would call if the file contained // such-and-such data. -type foo struct{} - type offlineModeTestParams struct { relayTestHelper t *testing.T @@ -181,7 +178,7 @@ func TestOfflineModeDeleteEnvironment(t *testing.T) { assert.Contains(t, keys, testFileDataEnv2.Params.SDKKey) _ = p.awaitEnvironment(testFileDataEnv1.Params.EnvID) - _ = p.awaitEnvironment(testFileDataEnv1.Params.EnvID) + _ = p.awaitEnvironment(testFileDataEnv2.Params.EnvID) p.updateHandler.DeleteEnvironment(testFileDataEnv1.Params.EnvID, testFileDataEnv1.Params.Identifiers.FilterKey) @@ -286,28 +283,22 @@ func TestOfflineModeSDKKeyCanExpire(t *testing.T) { cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(minimumCleanupInterval) offlineModeTest(t, cfg, func(p offlineModeTestParams) { + // It's important that the expiry be in the future (so that the key isn't ignored by the key rotator + // component), but it should also be in the near future so the test doesn't need to sleep long. + keyExpiry := time.Now().Add(10 * time.Millisecond) + update1 := RotateSDKKeyWithGracePeriod("key1", "key0", keyExpiry) + p.updateHandler.AddEnvironment(update1) - for i := 0; i < 3; i++ { - primary := config.SDKKey(fmt.Sprintf("key%v", i+1)) - expiring := config.SDKKey(fmt.Sprintf("key%v", i)) - - // It's important that the expiry be in the future (so that the key isn't ignored by the key rotator - // component), but it should also be in the near future so the test doesn't need to sleep long. - keyExpiry := time.Now().Add(10 * time.Millisecond) - update1 := RotateSDKKeyWithGracePeriod(primary, expiring, keyExpiry) - p.updateHandler.AddEnvironment(update1) - - // Waiting for the environment can take up to 1 second, but it could be much faster. In any case - // we'll still need to sleep at least the cleanup interval to ensure the key is expired. - env := p.awaitEnvironmentFor(update1.Params.EnvID, time.Second) - // Both the primary and the expiring key are in the accepted set until the expiry fires. - assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.ExpiringSDKKey.Key, update1.Params.EnvID}, env.GetCredentials()) - assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) - - assert.Eventually(t, func() bool { - return len(env.GetDeprecatedCredentials()) == 0 - }, time.Second, 10*time.Millisecond, "deprecated credentials should be cleaned up after expiry") - assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.EnvID}, env.GetCredentials()) - } + // Waiting for the environment can take up to 1 second, but it could be much faster. In any case + // we'll still need to sleep at least the cleanup interval to ensure the key is expired. + env := p.awaitEnvironmentFor(update1.Params.EnvID, time.Second) + // Both the primary and the expiring key are in the accepted set until the expiry fires. + assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.ExpiringSDKKey.Key, update1.Params.EnvID}, env.GetCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) + + assert.Eventually(t, func() bool { + return len(env.GetDeprecatedCredentials()) == 0 + }, time.Second, 10*time.Millisecond, "deprecated credentials should be cleaned up after expiry") + assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.EnvID}, env.GetCredentials()) }) } From ff6013ecb694c5a675e5c645a85c846c01f86a52 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Mon, 29 Jun 2026 15:08:09 -0700 Subject: [PATCH 25/66] test(concurrent-keys): integration tests for multi-key downstream auth (#725) Adds relay/concurrent_keys_auth_test.go, a dedicated integration suite for multi-key environments (sdkKeys[] / mobileKeys[]) where one anchor owns the single upstream connection. --- relay/concurrent_keys_auth_test.go | 539 +++++++++++++++++++++++++++++ 1 file changed, 539 insertions(+) create mode 100644 relay/concurrent_keys_auth_test.go diff --git a/relay/concurrent_keys_auth_test.go b/relay/concurrent_keys_auth_test.go new file mode 100644 index 00000000..d04b24e1 --- /dev/null +++ b/relay/concurrent_keys_auth_test.go @@ -0,0 +1,539 @@ +package relay + +// Downstream authentication for environments that accept multiple SDK keys and multiple mobile keys +// (the sdkKeys[]/mobileKeys[] array wire format), where a single anchor key owns the one upstream +// connection. These are permanent regression tests for that behavior. +// +// They complement — and deliberately do not duplicate — existing coverage: +// +// - Single-key / legacy expiring{}-slot rotation is covered by TestAutoConfigInitWithExpiringSDKKey, +// TestOfflineModeDeprecatedSDKKeyIsRespectedIfExpiryInFuture, TestOfflineModeSDKKeyCanExpire, etc. +// Those assert on the credential set via the legacy rotation path; these assert downstream +// authentication (and live stream connect/disconnect) via the array path. +// - Generic unknown-credential -> 401 is covered at the middleware/end-to-end layer; here we only +// test rejection that is specific to a multi-key environment (a credential outside a populated +// accepted set, and a key removed from the accepted set on reconcile). +// +// The tests reuse the in-process harnesses (autoConfTest for RAC, offlineModeTest for offline) so +// they get assertSDKEndpointsAvailability (auth accept/reject) and awaitClient/shouldNotCreateClient +// (proving the anchor owns the only upstream connection). + +import ( + "slices" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/filedata" + "github.com/launchdarkly/ld-relay/v8/internal/relayenv" + "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/configsource" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + "github.com/launchdarkly/eventsource" + "github.com/launchdarkly/go-configtypes" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + "github.com/launchdarkly/go-server-sdk-evaluation/v3/ldbuilders" + "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" + "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoretypes" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Shared identifiers for the multi-key environment used across these tests. The anchor is the key +// the singular sdkKey.value/mobKey points to; the "extra" keys are additional accepted entries that +// are NOT the anchor. +const ( + multiKeyEnvID = config.EnvironmentID("multikey-env") + anchorSDKKey = config.SDKKey("sdk-anchor") + extraSDKKey = config.SDKKey("sdk-extra") + anchorMobileKey = config.MobileKey("mob-anchor") + extraMobileKey = config.MobileKey("mob-extra") + multiKeyFlagKey = "multikey-flag" + + // rotatedAnchorSDKKey is a brand-new SDK key that becomes the anchor when the anchor is rotated. + rotatedAnchorSDKKey = config.SDKKey("sdk-rotated-anchor") +) + +var multiKeyIdentifiers = relayenv.EnvIdentifiers{ + ProjName: "Multi-Key Project", + ProjKey: "multikey-proj", + EnvName: "Multi-Key Env", + EnvKey: "multikey-env", +} + +// multiKeyEnvRep builds a RAC/offline EnvironmentRep for the multi-key env using the array wire +// format. The anchor is always anchorSDKKey/anchorMobileKey (singular sdkKey/mobKey); callers pass +// the full sdkKeys[]/mobileKeys[] arrays (which must include the anchor entry). +func multiKeyEnvRep(sdkKeys, mobileKeys []envfactory.ConcurrentKeyRep, version int) envfactory.EnvironmentRep { + return envfactory.EnvironmentRep{ + EnvID: multiKeyEnvID, + EnvKey: multiKeyIdentifiers.EnvKey, + EnvName: multiKeyIdentifiers.EnvName, + ProjKey: multiKeyIdentifiers.ProjKey, + ProjName: multiKeyIdentifiers.ProjName, + SDKKey: envfactory.SDKKeyRep{Value: anchorSDKKey}, + MobKey: anchorMobileKey, + SDKKeys: sdkKeys, + MobileKeys: mobileKeys, + Version: version, + } +} + +// defaultSDKKeyReps / defaultMobileKeyReps are the standard two-entry arrays (anchor + one extra, +// both permanent) used by the tests that don't involve expiry. +func defaultSDKKeyReps() []envfactory.ConcurrentKeyRep { + return []envfactory.ConcurrentKeyRep{ + {Key: "anchor-sdk", Value: string(anchorSDKKey)}, + {Key: "extra-sdk", Value: string(extraSDKKey)}, + } +} + +func defaultMobileKeyReps() []envfactory.ConcurrentKeyRep { + return []envfactory.ConcurrentKeyRep{ + {Key: "anchor-mob", Value: string(anchorMobileKey)}, + {Key: "extra-mob", Value: string(extraMobileKey)}, + } +} + +// multiKeyArchiveEnv builds the offline-mode ArchiveEnvironment equivalent of multiKeyEnvRep, +// carrying the accepted SDK/mobile key sets directly plus a single flag so the data store initializes. +func multiKeyArchiveEnv(sdkKeys []envfactory.AcceptedSDKKey, mobileKeys []envfactory.AcceptedMobileKey) filedata.ArchiveEnvironment { + return filedata.ArchiveEnvironment{ + Params: envfactory.EnvironmentParams{ + EnvID: multiKeyEnvID, + SDKKey: anchorSDKKey, + MobileKey: anchorMobileKey, + AcceptedSDKKeys: sdkKeys, + AcceptedMobileKeys: mobileKeys, + Identifiers: multiKeyIdentifiers, + }, + SDKData: multiKeySDKData(), + } +} + +func defaultAcceptedSDKKeys() []envfactory.AcceptedSDKKey { + return []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}, {Value: extraSDKKey}} +} + +func defaultAcceptedMobileKeys() []envfactory.AcceptedMobileKey { + return []envfactory.AcceptedMobileKey{{Value: anchorMobileKey}, {Value: extraMobileKey}} +} + +func multiKeySDKData() []ldstoretypes.Collection { + flag := ldbuilders.NewFlagBuilder(multiKeyFlagKey).Version(1).On(true).Build() + return []ldstoretypes.Collection{ + { + Kind: ldstoreimpl.Features(), + Items: []ldstoretypes.KeyedItemDescriptor{ + {Key: multiKeyFlagKey, Item: sharedtest.FlagDesc(flag)}, + }, + }, + } +} + +// rotatedAnchorRep returns the env rep after the anchor has rotated to newAnchor, keeping the +// existing non-anchor SDK key (extraSDKKey) accepted and the mobile keys unchanged. +func rotatedAnchorRep(newAnchor config.SDKKey, version int) envfactory.EnvironmentRep { + return envfactory.EnvironmentRep{ + EnvID: multiKeyEnvID, + EnvKey: multiKeyIdentifiers.EnvKey, + EnvName: multiKeyIdentifiers.EnvName, + ProjKey: multiKeyIdentifiers.ProjKey, + ProjName: multiKeyIdentifiers.ProjName, + SDKKey: envfactory.SDKKeyRep{Value: newAnchor}, + MobKey: anchorMobileKey, + SDKKeys: []envfactory.ConcurrentKeyRep{ + {Key: "rotated-anchor-sdk", Value: string(newAnchor)}, + {Key: "extra-sdk", Value: string(extraSDKKey)}, + }, + MobileKeys: defaultMobileKeyReps(), + Version: version, + } +} + +// A valid non-anchor key authenticates downstream; one upstream connection serves all accepted keys. + +func TestConcurrentKeysRAC_NonAnchorKeysAuthenticate(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + autoConfTest(t, testAutoConfDefaultConfig, &putEvent, func(p autoConfTestParams) { + // The anchor opens the single upstream client; no second client for the non-anchor key. + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) + + _ = p.awaitEnvironment(multiKeyEnvID) + + // Every accepted credential authenticates downstream, anchor and non-anchor alike. + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + }) +} + +func TestConcurrentKeysOffline_NonAnchorKeysAuthenticate(t *testing.T) { + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) + + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) + + env := p.awaitEnvironment(multiKeyEnvID) + + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + + // Flag data flows through the shared store that the single anchor connection populates. + flags, err := env.GetStore().GetAll(ldstoreimpl.Features()) + require.NoError(t, err) + assert.NotEmpty(t, flags) + }) +} + +// Per-credential rejection within a multi-key environment. + +func TestConcurrentKeysRAC_RejectsCredentialsOutsideAcceptedSet(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + autoConfTest(t, testAutoConfDefaultConfig, &putEvent, func(p autoConfTestParams) { + _ = p.awaitClient() + _ = p.awaitEnvironment(multiKeyEnvID) + + // (a) Accepted siblings authenticate; a credential outside the accepted set is rejected. + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(false, + config.SDKKey("sdk-not-accepted"), config.MobileKey("mob-not-accepted"), config.EnvironmentID("env-not-accepted")) + + // (b) Remove the extra keys via a patch that carries only the anchor; they must then be + // rejected, while the anchor (still accepted) keeps authenticating. + anchorOnly := multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{{Key: "anchor-sdk", Value: string(anchorSDKKey)}}, + []envfactory.ConcurrentKeyRep{{Key: "anchor-mob", Value: string(anchorMobileKey)}}, + 2, + ) + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(anchorOnly)) + + awaitCredentialRemoved(t, p.relay, extraSDKKey) + + p.assertSDKEndpointsAvailability(false, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + }) +} + +func TestConcurrentKeysOffline_RejectsCredentialsOutsideAcceptedSet(t *testing.T) { + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) + _ = p.awaitClient() + _ = p.awaitEnvironment(multiKeyEnvID) + + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(false, + config.SDKKey("sdk-not-accepted"), config.MobileKey("mob-not-accepted"), config.EnvironmentID("env-not-accepted")) + + // Reload with only the anchor accepted; the extra keys are dropped immediately (omitted). + p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv( + []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}}, + []envfactory.AcceptedMobileKey{{Value: anchorMobileKey}}, + )) + + awaitCredentialRemoved(t, p.relay, extraSDKKey) + + p.assertSDKEndpointsAvailability(false, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + }) +} + +// A connected SDK is disconnected when its (non-anchor) key expires. +// +// The downstream SDK connects on a non-anchor key while that key is still permanent, so the +// connection establishes independent of expiry timing. We then give the connected key a near-future +// expiry; once the timestamp passes, the cleanup ticker must drop the key AND disconnect the open +// stream. Covered for both an SDK key and a mobile key. (The live open-connection teardown is +// verified on the offline path, which uses a real SDK client that actually serves stream data; the +// RAC equivalent — TestConcurrentKeysRAC_KeyExpiryRemovesCredential — verifies the same expiry->reject +// outcome at the auth layer, since FakeLDClient does not serve stream data.) +func TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyExpires(t *testing.T) { + // The server-side stream (/all) emits "put"; the mobile streams (/meval, /mping) emit "ping". + run := func(t *testing.T, streamPath, firstEvent string, connectKey credential.SDKCredential, expiringSDK bool) { + cfg := config.Config{} + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) + offlineModeTest(t, cfg, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) + _ = p.awaitClient() + env := p.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + req := sharedtest.BuildRequestWithAuth("GET", streamPath, connectKey, nil) + sharedtest.WithStreamRequest(t, req, p.relay, func(eventCh <-chan eventsource.Event) { + // Confirm the stream is live before we expire the key. + sharedtest.AwaitEventOfType(t, eventCh, firstEvent, 5*time.Second) + + // Give the connected non-anchor key a near-future expiry; keep the anchor permanent. + expiry := time.Now().Add(100 * time.Millisecond) + sdkKeys := []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}, {Value: extraSDKKey}} + mobileKeys := []envfactory.AcceptedMobileKey{{Value: anchorMobileKey}, {Value: extraMobileKey}} + if expiringSDK { + sdkKeys[1].Expiry = expiry + } else { + mobileKeys[1].Expiry = expiry + } + p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv(sdkKeys, mobileKeys)) + + // The cleanup ticker drops the expired key and disconnects this stream. + awaitStreamClosed(t, eventCh, 5*time.Second) + }) + + // After expiry: the dropped key no longer authenticates; the anchor (sibling) still does. + if expiringSDK { + p.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") + } else { + p.assertSDKEndpointsAvailability(false, "", extraMobileKey, "") + } + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + }) + } + + t.Run("sdk key", func(t *testing.T) { run(t, "/all", "put", extraSDKKey, true) }) + t.Run("mobile key", func(t *testing.T) { + // base64 of {"key":"userkey","kind":"user"} — a valid context, not the legacy user format. + run(t, "/meval/eyJrZXkiOiJ1c2Vya2V5Iiwia2luZCI6InVzZXIifQ==", "ping", extraMobileKey, false) + }) +} + +func TestConcurrentKeysRAC_KeyExpiryRemovesCredential(t *testing.T) { + cfg := testAutoConfDefaultConfig + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + autoConfTest(t, cfg, &putEvent, func(p autoConfTestParams) { + _ = p.awaitClient() + _ = p.awaitEnvironment(multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + + // Patch: the non-anchor keys gain a near-future expiry; the anchor stays permanent. + expiry := time.Now().Add(100 * time.Millisecond).UnixMilli() + patch := multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-sdk", Value: string(anchorSDKKey)}, + {Key: "extra-sdk", Value: string(extraSDKKey), Expiry: msPtr(expiry)}, + }, + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-mob", Value: string(anchorMobileKey)}, + {Key: "extra-mob", Value: string(extraMobileKey), Expiry: msPtr(expiry)}, + }, + 2, + ) + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(patch)) + + // Once the expiry passes, the cleanup ticker drops the keys: they stop authenticating while + // the anchor is unaffected. + awaitCredentialRemoved(t, p.relay, extraSDKKey) + p.assertSDKEndpointsAvailability(false, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + }) +} + +// A connected SDK stays connected when its key gains a future expiry it hasn't reached yet. + +func TestConcurrentKeysOffline_ConnectionSurvivesWhenKeyGainsFutureExpiry(t *testing.T) { + cfg := config.Config{} + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) + offlineModeTest(t, cfg, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) + _ = p.awaitClient() + env := p.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + req := sharedtest.BuildRequestWithAuth("GET", "/all", extraSDKKey, nil) + sharedtest.WithStreamRequest(t, req, p.relay, func(eventCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) + + // Give the connected non-anchor key a FAR-future expiry; the cleanup ticker must not drop + // it during the test, so the open stream stays connected. + expiry := time.Now().Add(1 * time.Hour) + p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv( + []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}, {Value: extraSDKKey, Expiry: expiry}}, + defaultAcceptedMobileKeys(), + )) + + assertStreamStaysOpen(t, eventCh, 500*time.Millisecond) + }) + + // The key is now in the deprecated-but-accepted set (its expiry was applied) and still authenticates. + require.Eventually(t, func() bool { return credsContain(env.GetDeprecatedCredentials(), extraSDKKey) }, + time.Second, 5*time.Millisecond, "expiry was not applied to the connected key") + p.assertSDKEndpointsAvailability(true, extraSDKKey, "", "") + }) +} + +func TestConcurrentKeysRAC_KeyWithFutureExpiryStillAuthenticates(t *testing.T) { + cfg := testAutoConfDefaultConfig + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + autoConfTest(t, cfg, &putEvent, func(p autoConfTestParams) { + _ = p.awaitClient() + env := p.awaitEnvironment(multiKeyEnvID) + + // Patch: the non-anchor keys gain a FAR-future expiry; they remain accepted (not dropped). + expiry := time.Now().Add(1 * time.Hour).UnixMilli() + patch := multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-sdk", Value: string(anchorSDKKey)}, + {Key: "extra-sdk", Value: string(extraSDKKey), Expiry: msPtr(expiry)}, + }, + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-mob", Value: string(anchorMobileKey)}, + {Key: "extra-mob", Value: string(extraMobileKey), Expiry: msPtr(expiry)}, + }, + 2, + ) + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(patch)) + + // Confirm the expiry was applied (the key is now deprecated-but-accepted) and, after the + // cleanup ticker has had time to run, the future-dated key still authenticates. + require.Eventually(t, func() bool { return credsContain(env.GetDeprecatedCredentials(), extraSDKKey) }, + time.Second, 5*time.Millisecond, "future expiry was not applied") + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + }) +} + +// Rotating the anchor in the accepted set. +// +// Both tests run on the FakeLDClient harness, so they verify the routing/credential-level behavior +// of an anchor swap. The real-upstream store handover (avoiding an empty-store window) and +// rollback-on-init-failure robustness is the re-anchor work owned by T2.c. + +// When a new anchor arrives via RAC (sdkKey.value changes to a brand-new key), the upstream client +// swaps to the new anchor and the old anchor is dropped, while the non-anchor key stays accepted. +func TestConcurrentKeysRAC_RotatingAnchorUpdatesUpstreamClient(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + autoConfTest(t, testAutoConfDefaultConfig, &putEvent, func(p autoConfTestParams) { + client1 := p.awaitClient() + assert.Equal(t, anchorSDKKey, client1.Key) + _ = p.awaitEnvironment(multiKeyEnvID) + + // A new anchor arrives; the old anchor is rotated out, the non-anchor extra key is retained. + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(rotatedAnchorRep(rotatedAnchorSDKKey, 2))) + + // The new anchor opens the single upstream client; the old anchor's client closes. + client2 := p.awaitClient() + assert.Equal(t, rotatedAnchorSDKKey, client2.Key) + client1.AwaitClose(t, 5*time.Second) + + awaitCredentialRemoved(t, p.relay, anchorSDKKey) + + // The new anchor and the retained non-anchor key authenticate; the old anchor no longer does. + p.assertSDKEndpointsAvailability(true, rotatedAnchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, "", "") + p.assertSDKEndpointsAvailability(false, anchorSDKKey, "", "") + }) +} + +// A downstream SDK connected on a non-anchor key stays connected when the anchor is rotated out from +// under it. This uses a real (dummy) SDK client + RAC mock — rather than the FakeLDClient harness — +// because FakeLDClient never serves a put on the SSE stream, so it couldn't confirm the connection +// was actually established before rotating (and thus couldn't genuinely exercise "rotate while +// connected"). The connection-survival property holds even before the T2.c store-handover work, +// which addresses the empty-store data window during the swap, not connection drops. +func TestConcurrentKeysRAC_NonAnchorConnectionSurvivesAnchorRotation(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + racMock := configsource.NewRACMock(t, &putEvent) + + cfg := config.Config{AutoConfig: config.AutoConfigConfig{Key: testAutoConfKey}} + cfg.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(racMock.URL) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + relay, err := newRelayInternal(cfg, relayInternalOptions{ + loggers: mockLog.Loggers, + clientFactory: testclient.CreateDummyClient, + }) + require.NoError(t, err) + defer relay.Close() + + h := relayTestHelper{t: t, relay: relay} + env := h.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + // Connect a downstream SDK on the non-anchor key. + req := sharedtest.BuildRequestWithAuth("GET", "/all", extraSDKKey, nil) + sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { + // Confirm the non-anchor stream is live before rotating, so this genuinely exercises + // "rotate while connected" rather than racing the connection setup. + sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) + + // Rotate the anchor to a brand-new key while the non-anchor stream is connected. + racMock.Send(configsource.MakeAutoConfigPatchEvent(rotatedAnchorRep(rotatedAnchorSDKKey, 2))) + + // Wait for the rotation to take effect: the new anchor resolves, the old anchor is gone. + require.Eventually(t, func() bool { + _, errNew := relay.getEnvironment(sdkauth.New(rotatedAnchorSDKKey)) + _, errOld := relay.getEnvironment(sdkauth.New(anchorSDKKey)) + return errNew == nil && errOld != nil + }, 5*time.Second, 5*time.Millisecond) + + // The non-anchor key's open stream is not disconnected by the swap (a duplicate put may + // arrive from the new anchor's initial sync — that's fine; only a close would fail here). + assertStreamStaysOpen(t, eventCh, 500*time.Millisecond) + }) + + // The non-anchor key still authenticates after the rotation. + h.assertSDKEndpointsAvailability(true, extraSDKKey, "", "") +} + +// awaitStreamClosed reads from a WithStreamRequest event channel until the stream-closed sentinel +// (a nil event) arrives, failing if the timeout elapses first. Non-nil events are ignored. +func awaitStreamClosed(t *testing.T, eventCh <-chan eventsource.Event, timeout time.Duration) { + t.Helper() + deadline := time.NewTimer(timeout) + defer deadline.Stop() + for { + select { + case e := <-eventCh: + if e == nil { + return // stream closed — the disconnect we expect + } + case <-deadline.C: + t.Fatalf("timed out after %s waiting for the stream to be disconnected", timeout) + return + } + } +} + +// assertStreamStaysOpen verifies that no stream-closed sentinel arrives within the given window. +func assertStreamStaysOpen(t *testing.T, eventCh <-chan eventsource.Event, window time.Duration) { + t.Helper() + deadline := time.NewTimer(window) + defer deadline.Stop() + for { + select { + case e := <-eventCh: + if e == nil { + t.Fatalf("stream was unexpectedly disconnected within %s", window) + return + } + case <-deadline.C: + return // still open after the window — as expected + } + } +} + +func msPtr(v int64) *int64 { return &v } + +func credsContain(creds []credential.SDKCredential, target credential.SDKCredential) bool { + return slices.Contains(creds, target) +} + +// awaitCredentialRemoved blocks until the given credential no longer resolves to an environment. +func awaitCredentialRemoved(t *testing.T, relay *Relay, cred credential.SDKCredential) { + t.Helper() + require.Eventually(t, func() bool { + _, err := relay.getEnvironment(sdkauth.New(cred)) + return err != nil + }, time.Second, 5*time.Millisecond, "credential was not removed from the accepted set") +} From 7f3c0c4264f2747ca222f28dc98ba4ae55426415 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 30 Jun 2026 09:51:27 -0700 Subject: [PATCH 26/66] fix(envfactory): reject primary mobile key absent from mobileKeys[] (#728) Reject a defined primary mobile key (the wire's mobKey) that is absent from mobileKeys[], mirroring the existing anchor-not-in-array invariant. --- internal/credential/accepted_set.go | 22 ++++++++++++++------ internal/envfactory/reconcile_helper.go | 12 +++++++++++ internal/envfactory/reconcile_helper_test.go | 21 +++++++++++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/internal/credential/accepted_set.go b/internal/credential/accepted_set.go index 201bd9e6..19aa2a74 100644 --- a/internal/credential/accepted_set.go +++ b/internal/credential/accepted_set.go @@ -19,8 +19,8 @@ import ( // // WithAnchor / WithPrimaryMobileKey both add the key to the set and designate it, so adding a // single key takes one call. Build requires that an anchor was designated. (Structural validation of -// the wire payload — undefined credentials, an anchor absent from the array — happens upstream when -// the payload is parsed into the set; see SDK-2547.) +// the wire payload — undefined credentials, a designated key absent from its array — happens upstream +// when the payload is parsed into the set.) // // A key's expiry is taken from its entry in this set; the legacy sdkKey.expiring{} wire slot is not // consulted when building it. @@ -55,11 +55,13 @@ func (s AcceptedSet) hasMobileKey(key config.MobileKey) bool { var errAcceptedSetMissingSDKKey = errors.New("accepted credential set must contain at least one SDK key") // MalformedCredentialSetError is returned when a credential payload cannot produce a valid -// AcceptedSet. This covers two cases: +// AcceptedSet. This covers: // -// 1. The anchor SDK key (sdkKey.value) is absent or undefined — a violation of the backend -// invariant that an anchor is always designated. -// 2. An entry in sdkKeys[] or mobileKeys[] has an empty value — a credential that would be +// 1. The anchor SDK key (sdkKey.value) is absent or undefined, or defined but not present in +// sdkKeys[] — a violation of the invariant that the designated anchor is one of the accepted keys. +// 2. The primary mobile key (mobKey) is defined but not present in mobileKeys[] — the mobile-key +// analogue of the anchor invariant. +// 3. An entry in sdkKeys[] or mobileKeys[] has an empty value — a credential that would be // accepted by relay but can never authenticate any SDK. // // Validation happens before Reconcile is called; Rotator.Reconcile trusts the set it is handed. @@ -89,6 +91,14 @@ func NewAnchorNotInSetError() *MalformedCredentialSetError { return &MalformedCredentialSetError{msg: "malformed credential set: anchor SDK key is not present in sdkKeys[]"} } +// NewPrimaryMobileKeyNotInSetError returns a MalformedCredentialSetError for a payload whose +// designated primary mobile key (mobKey) is defined but not present in the mobileKeys[] array — the +// mobile-key analogue of NewAnchorNotInSetError. The key value is a secret, so it is deliberately not +// included in the message. +func NewPrimaryMobileKeyNotInSetError() *MalformedCredentialSetError { + return &MalformedCredentialSetError{msg: "malformed credential set: primary mobile key is not present in mobileKeys[]"} +} + // NewEmptyCredentialError returns a MalformedCredentialSetError for a key-array entry whose // value field is empty. kind is "sdkKeys" or "mobileKeys"; identifier is the key's identifier // string (may be empty for old-format payloads that synthesize from the singular fields). diff --git a/internal/envfactory/reconcile_helper.go b/internal/envfactory/reconcile_helper.go index 2b262fdb..c20c937a 100644 --- a/internal/envfactory/reconcile_helper.go +++ b/internal/envfactory/reconcile_helper.go @@ -66,10 +66,14 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. return credential.AcceptedSet{}, anchor, credential.NewAnchorNotInSetError() } + primaryMobileInArray := false for _, k := range params.AcceptedMobileKeys { if !k.Value.Defined() { return credential.AcceptedSet{}, anchor, credential.NewEmptyCredentialError("mobileKeys", k.Key) } + if k.Value == params.MobileKey { + primaryMobileInArray = true + } if k.Expiry.IsZero() { b.WithMobileKey(k.Value) } else { @@ -77,6 +81,14 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. } } + // The primary mobile key, when the environment has one, must be in mobileKeys[] — the mobile + // analogue of the anchor invariant above. A defined mobKey absent from the array is malformed: + // without this guard the primary would be silently left undesignated, clearing it on reconcile and + // breaking event forwarding. (An undefined mobKey is valid — a server-side-only environment.) + if params.MobileKey.Defined() && !primaryMobileInArray { + return credential.AcceptedSet{}, anchor, credential.NewPrimaryMobileKeyNotInSetError() + } + set, err := b.Build() if err != nil { return credential.AcceptedSet{}, anchor, err diff --git a/internal/envfactory/reconcile_helper_test.go b/internal/envfactory/reconcile_helper_test.go index 2ab9a82e..b3abff11 100644 --- a/internal/envfactory/reconcile_helper_test.go +++ b/internal/envfactory/reconcile_helper_test.go @@ -176,6 +176,27 @@ func TestBuildAcceptedSet_AnchorNotInArray(t *testing.T) { assert.Contains(t, malformed.Error(), "not present in sdkKeys[]") } +// TestBuildAcceptedSet_PrimaryMobileNotInArray verifies the mobile analogue of the anchor invariant: +// a defined mobKey absent from mobileKeys[] is rejected. Without this guard the primary mobile key +// would be silently left undesignated, clearing it on reconcile and breaking event forwarding. +func TestBuildAcceptedSet_PrimaryMobileNotInArray(t *testing.T) { + params := EnvironmentParams{ + EnvID: "env-abc", + SDKKey: "sdk-anchor", + MobileKey: "mob-primary", // defined... + AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, + AcceptedMobileKeys: []AcceptedMobileKey{ + {Key: "other", Value: "mob-other"}, // ...but NOT in the array + }, + } + _, _, err := BuildAcceptedSet(params) + + require.Error(t, err) + var malformed *credential.MalformedCredentialSetError + require.True(t, errors.As(err, &malformed)) + assert.Contains(t, malformed.Error(), "not present in mobileKeys[]") +} + // TestBuildAcceptedSet_NoMobileKey verifies that an environment with no mobile key (e.g. a // server-side-only environment) is valid: ToParams must not synthesize a phantom empty mobileKeys // entry that BuildAcceptedSet would reject as malformed. From 42060b1c04ad5c0ad04d2b0a6b64303f3fea3789 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 30 Jun 2026 09:49:18 -0700 Subject: [PATCH 27/66] test: use a valid context instead of legacy user format in relay tests (#727) Updates two relay test fixtures that still modeled the eval context they send to the SDK endpoints as a legacy LDUser ({"key":"userkey"}, no kind) to use a valid single-kind evaluation context ({"key":"userkey","kind":"user"}). --- internal/sharedtest/endpoints.go | 4 ++-- relay/testutils_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/sharedtest/endpoints.go b/internal/sharedtest/endpoints.go index 8655d537..e42ab44a 100644 --- a/internal/sharedtest/endpoints.go +++ b/internal/sharedtest/endpoints.go @@ -13,8 +13,8 @@ import ( // that configures Relay's endpoint routing, so that if we accidentally change the routing, the tests // will fail, rather than succeed based on incorrect paths. -// SimpleUserJSON is a basic user. -const SimpleUserJSON = `{"key":"userkey"}` +// SimpleUserJSON is a basic single-kind context. +const SimpleUserJSON = `{"key":"userkey","kind":"user"}` // ToBase64 is a shortcut for base64 encoding. func ToBase64(s string) string { diff --git a/relay/testutils_test.go b/relay/testutils_test.go index 80ed2b6f..732ad6b7 100644 --- a/relay/testutils_test.go +++ b/relay/testutils_test.go @@ -81,8 +81,8 @@ func (h relayTestHelper) assertSDKEndpointsAvailability( // appropriate HTTP status. These are tested more thoroughly in test suites like DoStreamEndpointsTests, // but we use this simpler test when we're dynamically changing what the credentials are. - simpleUserJSON := []byte(`{"key":"userkey"}`) - simpleUserBase64 := "eyJrZXkiOiJ1c2Vya2V5In0=" + simpleUserJSON := []byte(`{"key":"userkey","kind":"user"}`) + simpleUserBase64 := "eyJrZXkiOiJ1c2Vya2V5Iiwia2luZCI6InVzZXIifQ==" status200Or401, status200Or404 := 200, 200 if !shouldBeAvailable { status200Or401 = 401 From 46320e8ddf806e124c4d2b666dc021adc7e5aed5 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 30 Jun 2026 13:13:47 -0700 Subject: [PATCH 28/66] refactor(credential): carry wire key identifiers through the accepted set (#729) Extends the credential accepted-set model so each SDK/mobile key stores optional wire key identifiers (plus expiry) alongside the secret value, preparing later /status exposure without changing auth behavior in this PR. --- internal/credential/accepted_set.go | 21 ++--- internal/credential/accepted_set_builder.go | 92 +++++++++---------- .../credential/accepted_set_builder_test.go | 16 ++-- internal/credential/rotator.go | 63 +++++++------ internal/credential/rotator_test.go | 45 ++++----- internal/envfactory/reconcile_helper.go | 36 +++----- internal/envfactory/reconcile_helper_test.go | 83 +++++++++-------- internal/relayenv/env_context_impl_test.go | 42 ++++----- .../relayenv/env_context_reanchor_test.go | 7 +- internal/util/pointer.go | 11 +++ internal/util/pointer_test.go | 32 +++++++ 11 files changed, 241 insertions(+), 207 deletions(-) create mode 100644 internal/util/pointer.go create mode 100644 internal/util/pointer_test.go diff --git a/internal/credential/accepted_set.go b/internal/credential/accepted_set.go index 19aa2a74..9366ae30 100644 --- a/internal/credential/accepted_set.go +++ b/internal/credential/accepted_set.go @@ -3,7 +3,6 @@ package credential import ( "errors" "fmt" - "time" "github.com/launchdarkly/ld-relay/v8/config" ) @@ -27,12 +26,12 @@ import ( // // Construct an AcceptedSet with AcceptedSetBuilder (see accepted_set_builder.go). type AcceptedSet struct { - // sdkKeys and mobileKeys store each accepted key once, keyed by value, so duplicates collapse - // without a containment scan. The map value is the key's expiry: a nil *time.Time means the key - // is permanent. A nil map is a valid empty set (reads return absent; only the builder writes). - sdkKeys map[config.SDKKey]*time.Time + // sdkKeys and mobileKeys store each accepted key once, keyed by value (the secret), so duplicates + // collapse without a containment scan. The map value carries the key's metadata (see + // acceptedKeyInfo). A nil map is a valid empty set (reads return absent; only the builder writes). + sdkKeys map[config.SDKKey]acceptedKeyInfo anchor config.SDKKey - mobileKeys map[config.MobileKey]*time.Time + mobileKeys map[config.MobileKey]acceptedKeyInfo primaryMobileKey config.MobileKey envID config.EnvironmentID } @@ -100,15 +99,15 @@ func NewPrimaryMobileKeyNotInSetError() *MalformedCredentialSetError { } // NewEmptyCredentialError returns a MalformedCredentialSetError for a key-array entry whose -// value field is empty. kind is "sdkKeys" or "mobileKeys"; identifier is the key's identifier -// string (may be empty for old-format payloads that synthesize from the singular fields). -func NewEmptyCredentialError(kind, identifier string) *MalformedCredentialSetError { - if identifier == "" { +// value field is empty. kind is "sdkKeys" or "mobileKeys"; key is the entry's wire "key" identifier +// (may be empty for old-format payloads that synthesize from the singular fields). +func NewEmptyCredentialError(kind, key string) *MalformedCredentialSetError { + if key == "" { return &MalformedCredentialSetError{ msg: fmt.Sprintf("malformed credential set: %s entry has an empty value", kind), } } return &MalformedCredentialSetError{ - msg: fmt.Sprintf("malformed credential set: %s entry %q has an empty value", kind, identifier), + msg: fmt.Sprintf("malformed credential set: %s entry %q has an empty value", kind, key), } } diff --git a/internal/credential/accepted_set_builder.go b/internal/credential/accepted_set_builder.go index 427f2943..d3b4b561 100644 --- a/internal/credential/accepted_set_builder.go +++ b/internal/credential/accepted_set_builder.go @@ -16,77 +16,69 @@ type AcceptedSetBuilder struct { func NewAcceptedSetBuilder() *AcceptedSetBuilder { return &AcceptedSetBuilder{ set: AcceptedSet{ - sdkKeys: make(map[config.SDKKey]*time.Time), - mobileKeys: make(map[config.MobileKey]*time.Time), + sdkKeys: make(map[config.SDKKey]acceptedKeyInfo), + mobileKeys: make(map[config.MobileKey]acceptedKeyInfo), }, } } -// WithSDKKey adds a permanent (non-expiring) SDK key. It is a no-op if the key is undefined or -// already present. -func (b *AcceptedSetBuilder) WithSDKKey(key config.SDKKey) *AcceptedSetBuilder { - b.addSDKKey(key, nil) - return b +// SDKKeyParams describes one accepted server-side SDK key for the builder: the credential value plus +// the optional wire "key" identifier (nil when absent) and optional expiry (nil = permanent). +type SDKKeyParams struct { + Value config.SDKKey + Key *string + Expiry *time.Time } -// WithExpiringSDKKey adds an SDK key that should be accepted until the given expiry. It is a no-op -// if the key is undefined or already present. -func (b *AcceptedSetBuilder) WithExpiringSDKKey(key config.SDKKey, expiry time.Time) *AcceptedSetBuilder { - b.addSDKKey(key, &expiry) - return b +// MobileKeyParams describes one accepted mobile key for the builder. See SDKKeyParams. +type MobileKeyParams struct { + Value config.MobileKey + Key *string + Expiry *time.Time } -// WithAnchor adds key (if not already present) and designates it as the anchor — the SDK key -// that owns the environment's upstream connection. It is a no-op if the key is undefined. -func (b *AcceptedSetBuilder) WithAnchor(key config.SDKKey) *AcceptedSetBuilder { - if key.Defined() { - b.addSDKKey(key, nil) - b.set.anchor = key +// WithSDKKey adds a server-side SDK key. It is a no-op if the value is undefined or already present +// (the first metadata recorded for a value wins). +func (b *AcceptedSetBuilder) WithSDKKey(p SDKKeyParams) *AcceptedSetBuilder { + if !p.Value.Defined() || b.set.hasSDKKey(p.Value) { + return b } + b.set.sdkKeys[p.Value] = acceptedKeyInfo{key: p.Key, expiry: p.Expiry} return b } -// addSDKKey records the key with the given expiry (nil = permanent), skipping undefined keys and -// keys already in the set (the first expiry recorded for a key wins). -func (b *AcceptedSetBuilder) addSDKKey(key config.SDKKey, expiry *time.Time) { - if !key.Defined() || b.set.hasSDKKey(key) { - return +// WithAnchor adds p.Value and designates it as the anchor — the SDK key that owns the environment's +// upstream connection. The anchor is always permanent, so p.Expiry is ignored. It is a no-op if the +// value is undefined. Unlike WithSDKKey it overwrites any existing entry for the value, since +// designating the anchor takes precedence over an earlier non-anchor add. +func (b *AcceptedSetBuilder) WithAnchor(p SDKKeyParams) *AcceptedSetBuilder { + if !p.Value.Defined() { + return b } - b.set.sdkKeys[key] = expiry -} - -// WithMobileKey adds a permanent (non-expiring) mobile key. It is a no-op if the key is undefined or -// already present. -func (b *AcceptedSetBuilder) WithMobileKey(key config.MobileKey) *AcceptedSetBuilder { - b.addMobileKey(key, nil) + b.set.sdkKeys[p.Value] = acceptedKeyInfo{key: p.Key, expiry: nil} + b.set.anchor = p.Value return b } -// WithExpiringMobileKey adds a mobile key that should be accepted until the given expiry. It is a -// no-op if the key is undefined or already present. -func (b *AcceptedSetBuilder) WithExpiringMobileKey(key config.MobileKey, expiry time.Time) *AcceptedSetBuilder { - b.addMobileKey(key, &expiry) - return b -} - -// WithPrimaryMobileKey adds key (if not already present) and designates it as the primary mobile -// key — the singular default (the wire's mobKey) used where one mobile key is required, e.g. event -// forwarding. It is a no-op if the key is undefined. -func (b *AcceptedSetBuilder) WithPrimaryMobileKey(key config.MobileKey) *AcceptedSetBuilder { - if key.Defined() { - b.addMobileKey(key, nil) - b.set.primaryMobileKey = key +// WithMobileKey adds a mobile key. It is a no-op if the value is undefined or already present. +func (b *AcceptedSetBuilder) WithMobileKey(p MobileKeyParams) *AcceptedSetBuilder { + if !p.Value.Defined() || b.set.hasMobileKey(p.Value) { + return b } + b.set.mobileKeys[p.Value] = acceptedKeyInfo{key: p.Key, expiry: p.Expiry} return b } -// addMobileKey records the key with the given expiry (nil = permanent), skipping undefined keys and -// keys already in the set (the first expiry recorded for a key wins). -func (b *AcceptedSetBuilder) addMobileKey(key config.MobileKey, expiry *time.Time) { - if !key.Defined() || b.set.hasMobileKey(key) { - return +// WithPrimaryMobileKey adds p.Value and designates it as the primary mobile key — the singular +// default (the wire's mobKey) used where one mobile key is required, e.g. event forwarding. The +// primary is always permanent, so p.Expiry is ignored. It is a no-op if the value is undefined. +func (b *AcceptedSetBuilder) WithPrimaryMobileKey(p MobileKeyParams) *AcceptedSetBuilder { + if !p.Value.Defined() { + return b } - b.set.mobileKeys[key] = expiry + b.set.mobileKeys[p.Value] = acceptedKeyInfo{key: p.Key, expiry: nil} + b.set.primaryMobileKey = p.Value + return b } // WithEnvironmentID sets the environment ID. It is a no-op if the ID is undefined. diff --git a/internal/credential/accepted_set_builder_test.go b/internal/credential/accepted_set_builder_test.go index 4ef837d1..3f89f9c9 100644 --- a/internal/credential/accepted_set_builder_test.go +++ b/internal/credential/accepted_set_builder_test.go @@ -12,18 +12,18 @@ import ( func TestAcceptedSetBuilderValidation(t *testing.T) { // No SDK key at all is a caller error. _, err := NewAcceptedSetBuilder(). - WithMobileKey(config.MobileKey("mob")). + WithMobileKey(MobileKeyParams{Value: "mob"}). WithEnvironmentID(config.EnvironmentID("env")). Build() require.ErrorIs(t, err, errAcceptedSetMissingSDKKey) // An SDK key with no designated anchor is malformed. var malformed *MalformedCredentialSetError - _, err = NewAcceptedSetBuilder().WithSDKKey(config.SDKKey("sdk")).Build() + _, err = NewAcceptedSetBuilder().WithSDKKey(SDKKeyParams{Value: "sdk"}).Build() require.ErrorAs(t, err, &malformed) // WithAnchor adds the key and designates it as the anchor, so Build succeeds. - set, err := NewAcceptedSetBuilder().WithAnchor(config.SDKKey("sdk")).Build() + set, err := NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: "sdk"}).Build() require.NoError(t, err) assert.True(t, set.hasSDKKey(config.SDKKey("sdk"))) assert.Equal(t, config.SDKKey("sdk"), set.anchor) @@ -32,11 +32,11 @@ func TestAcceptedSetBuilderValidation(t *testing.T) { func TestAcceptedSetBuilderDeduplicates(t *testing.T) { // Adding the same key more than once (including via WithPrimary*) keeps a single entry. set := mustBuild(t, NewAcceptedSetBuilder(). - WithSDKKey(config.SDKKey("sdk")). - WithAnchor(config.SDKKey("sdk")). - WithSDKKey(config.SDKKey("sdk")). - WithMobileKey(config.MobileKey("mob")). - WithPrimaryMobileKey(config.MobileKey("mob"))) + WithSDKKey(SDKKeyParams{Value: "sdk"}). + WithAnchor(SDKKeyParams{Value: "sdk"}). + WithSDKKey(SDKKeyParams{Value: "sdk"}). + WithMobileKey(MobileKeyParams{Value: "mob"}). + WithPrimaryMobileKey(MobileKeyParams{Value: "mob"})) assert.Len(t, set.sdkKeys, 1) assert.Len(t, set.mobileKeys, 1) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 728305ad..887e064d 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -11,6 +11,7 @@ import ( // acceptedKeyInfo holds per-key metadata for the accepted-set maps. type acceptedKeyInfo struct { expiry *time.Time // nil = permanent + key *string // wire "key" identifier — non-secret human-readable name; nil when absent } type Rotator struct { @@ -28,11 +29,11 @@ type Rotator struct { // acceptedSDKKeys is the full set of accepted SDK keys with optional per-key expiry. // A nil expiry means the key is permanent. The anchor is always present with a nil expiry. - acceptedSDKKeys map[config.SDKKey]*acceptedKeyInfo + acceptedSDKKeys map[config.SDKKey]acceptedKeyInfo // acceptedMobileKeys is the full set of accepted mobile keys with optional per-key expiry. // A nil expiry means the key is permanent. - acceptedMobileKeys map[config.MobileKey]*acceptedKeyInfo + acceptedMobileKeys map[config.MobileKey]acceptedKeyInfo expirations []SDKCredential additions []SDKCredential @@ -51,8 +52,8 @@ type InitialCredentials struct { func NewRotator(loggers ldlog.Loggers) *Rotator { r := &Rotator{ loggers: loggers, - acceptedSDKKeys: make(map[config.SDKKey]*acceptedKeyInfo), - acceptedMobileKeys: make(map[config.MobileKey]*acceptedKeyInfo), + acceptedSDKKeys: make(map[config.SDKKey]acceptedKeyInfo), + acceptedMobileKeys: make(map[config.MobileKey]acceptedKeyInfo), } return r } @@ -70,10 +71,10 @@ func (r *Rotator) Initialize(credentials []SDKCredential) { switch cred := cred.(type) { case config.SDKKey: r.anchorKey = cred - r.acceptedSDKKeys[cred] = &acceptedKeyInfo{} + r.acceptedSDKKeys[cred] = acceptedKeyInfo{} case config.MobileKey: r.primaryMobileKey = cred - r.acceptedMobileKeys[cred] = &acceptedKeyInfo{} + r.acceptedMobileKeys[cred] = acceptedKeyInfo{} case config.EnvironmentID: r.primaryEnvironmentID = cred } @@ -222,23 +223,23 @@ type reconcilableKey interface { // key no longer desired is dropped and queued as an expiration. Per-key expiry is stored as data on // the accepted entry; the cleanup ticker is what later acts on it. The caller must hold the write lock. func reconcileAcceptedKeys[K reconcilableKey]( - desired map[K]*time.Time, - accepted map[K]*acceptedKeyInfo, + desired map[K]acceptedKeyInfo, + accepted map[K]acceptedKeyInfo, additions *[]SDKCredential, expirations *[]SDKCredential, loggers ldlog.Loggers, kind string, ) { - // First pass: walk every key the set wants us to accept. If we already accept it, just refresh - // its expiry; if it's new, start accepting it and queue it as an addition. - for key, expiry := range desired { - if info, ok := accepted[key]; ok { - info.expiry = expiry - } else { - accepted[key] = &acceptedKeyInfo{expiry: expiry} + // First pass: walk every key the set wants us to accept. Writing the desired entry over the + // accepted one refreshes its metadata — both the expiry and the wire "key" identifier, clearing + // the identifier when the new payload carries none so a stale name never lingers in /status. A key + // we don't yet accept is also queued as an addition. + for key, want := range desired { + if _, ok := accepted[key]; !ok { *additions = append(*additions, key) loggers.Infof("%s %s is now accepted", kind, key.Masked()) } + accepted[key] = want } // Second pass: walk every key we currently accept and drop the ones the set no longer wants. // Keys still desired were handled above, so skip them; the rest are revoked outright (removed @@ -254,34 +255,32 @@ func reconcileAcceptedKeys[K reconcilableKey]( } // reconcileSDKKeys diffs the desired SDK keys against the accepted set and applies the result via -// reconcileAcceptedKeys. The anchor is always accepted and permanent, regardless of any expiry the -// payload may carry for it. The caller must hold the write lock. +// reconcileAcceptedKeys. The set is trusted as well-formed: BuildAcceptedSet / the builder guarantee +// the anchor is present and permanent (WithAnchor forces a nil expiry), so no special handling is +// needed here. The caller must hold the write lock. func (r *Rotator) reconcileSDKKeys(set AcceptedSet, anchor config.SDKKey, now time.Time) { - desired := make(map[config.SDKKey]*time.Time, len(set.sdkKeys)) - for key, expiry := range set.sdkKeys { - if expiry != nil && !now.Before(*expiry) { + desired := make(map[config.SDKKey]acceptedKeyInfo, len(set.sdkKeys)) + for key, info := range set.sdkKeys { + if info.expiry != nil && !now.Before(*info.expiry) { continue // already expired; treat as absent } - desired[key] = expiry + desired[key] = info } - desired[anchor] = nil reconcileAcceptedKeys(desired, r.acceptedSDKKeys, &r.additions, &r.expirations, r.loggers, "SDK key") r.anchorKey = anchor } -// reconcileMobileKeys mirrors reconcileSDKKeys for mobile keys. The primary mobile key — the wire's -// singular mobKey, used where one mobile key is required (e.g. event forwarding) — is always accepted -// and permanent; an empty value means the set declared no mobile key. The caller must hold the lock. +// reconcileMobileKeys mirrors reconcileSDKKeys for mobile keys. The set is trusted as well-formed: +// when a primary mobile key is designated, the builder guarantees it is present and permanent +// (WithPrimaryMobileKey forces a nil expiry). An empty primary means the set declared no mobile key. +// The caller must hold the lock. func (r *Rotator) reconcileMobileKeys(set AcceptedSet, now time.Time) { - desired := make(map[config.MobileKey]*time.Time, len(set.mobileKeys)) - for key, expiry := range set.mobileKeys { - if expiry != nil && !now.Before(*expiry) { + desired := make(map[config.MobileKey]acceptedKeyInfo, len(set.mobileKeys)) + for key, info := range set.mobileKeys { + if info.expiry != nil && !now.Before(*info.expiry) { continue // already expired; treat as absent } - desired[key] = expiry - } - if set.primaryMobileKey.Defined() { - desired[set.primaryMobileKey] = nil + desired[key] = info } reconcileAcceptedKeys(desired, r.acceptedMobileKeys, &r.additions, &r.expirations, r.loggers, "Mobile key") r.primaryMobileKey = set.primaryMobileKey diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index b682c8f2..91238134 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -6,6 +6,7 @@ import ( "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -53,7 +54,7 @@ func TestReconcileAnchorOnly(t *testing.T) { anchor := config.SDKKey("anchor") now := time.Now() - r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor)), now) + r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor})), now) additions, expirations := r.StepTime(now) assert.ElementsMatch(t, []SDKCredential{anchor}, additions) @@ -70,7 +71,7 @@ func TestReconcileMultipleSDKKeys(t *testing.T) { now := time.Now() r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor).WithSDKKey(other)), now) + mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor}).WithSDKKey(SDKKeyParams{Value: other})), now) additions, expirations := r.StepTime(now) // Both server keys are accepted; only the anchor is primary. @@ -89,7 +90,7 @@ func TestReconcileMultipleMobileKeys(t *testing.T) { now := time.Now() r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor).WithPrimaryMobileKey(mob1).WithMobileKey(mob2)), now) + mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor}).WithPrimaryMobileKey(MobileKeyParams{Value: mob1}).WithMobileKey(MobileKeyParams{Value: mob2})), now) additions, _ := r.StepTime(now) // Every mobile key is accepted; the designated one is the primary. @@ -106,11 +107,11 @@ func TestReconcileRevokesOmittedKeys(t *testing.T) { now := time.Now() r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor).WithSDKKey(other).WithPrimaryMobileKey(mob)), now) + mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor}).WithSDKKey(SDKKeyParams{Value: other}).WithPrimaryMobileKey(MobileKeyParams{Value: mob})), now) r.StepTime(now) // Reconciling to just the anchor revokes the omitted server and mobile keys. - r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(anchor)), now) + r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor})), now) additions, expirations := r.StepTime(now) assert.Empty(t, additions) @@ -132,10 +133,10 @@ func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { r.Reconcile( mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(anchor). - WithExpiringSDKKey(expiringSDK, now.Add(time.Hour)). - WithPrimaryMobileKey(mob). - WithExpiringMobileKey(expiringMobile, now.Add(time.Hour))), + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: expiringSDK, Expiry: util.PtrOrNil(now.Add(time.Hour))}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob}). + WithMobileKey(MobileKeyParams{Value: expiringMobile, Expiry: util.PtrOrNil(now.Add(time.Hour))})), now) additions, expirations := r.StepTime(now) @@ -158,9 +159,9 @@ func TestReconcilePrimaryMobileKeyIsAlwaysAccepted(t *testing.T) { now := time.Unix(1000, 0) set := mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(anchor). - WithExpiringMobileKey(mob, now.Add(-time.Hour)). // already expired in the payload... - WithPrimaryMobileKey(mob)) // ...but designated as the primary + WithAnchor(SDKKeyParams{Value: anchor}). + WithMobileKey(MobileKeyParams{Value: mob, Expiry: util.PtrOrNil(now.Add(-time.Hour))}). // already expired in the payload... + WithPrimaryMobileKey(MobileKeyParams{Value: mob})) // ...but designated as the primary r.Reconcile(set, now) r.StepTime(now) @@ -184,10 +185,10 @@ func TestReconcileExpiringKeysAreEvictedByStepTime(t *testing.T) { r.Reconcile( mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(anchor). - WithExpiringSDKKey(expiringSDK, expiry). - WithPrimaryMobileKey(mob). - WithExpiringMobileKey(expiringMobile, expiry)), + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: expiringSDK, Expiry: util.PtrOrNil(expiry)}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob}). + WithMobileKey(MobileKeyParams{Value: expiringMobile, Expiry: util.PtrOrNil(expiry)})), now) additions, expirations := r.StepTime(now) require.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, additions) @@ -218,8 +219,8 @@ func TestReconcileAlreadyExpiredKeyIsIgnoredOnAdd(t *testing.T) { r.Reconcile( mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(anchor). - WithExpiringSDKKey(staleKey, alreadyExpired)), + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: staleKey, Expiry: util.PtrOrNil(alreadyExpired)})), now) additions, expirations := r.StepTime(now) @@ -242,8 +243,8 @@ func TestReconcileDeExpiryRestoresKey(t *testing.T) { // First reconcile: key is accepted with a future expiry. r.Reconcile( mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(anchor). - WithExpiringSDKKey(key, expiry)), + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: key, Expiry: util.PtrOrNil(expiry)})), now) r.StepTime(now) require.ElementsMatch(t, []SDKCredential{key}, r.DeprecatedCredentials()) @@ -251,8 +252,8 @@ func TestReconcileDeExpiryRestoresKey(t *testing.T) { // Second reconcile: same key, no expiry (de-expiry). r.Reconcile( mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(anchor). - WithSDKKey(key)), + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: key})), now) r.StepTime(now) diff --git a/internal/envfactory/reconcile_helper.go b/internal/envfactory/reconcile_helper.go index c20c937a..0f117736 100644 --- a/internal/envfactory/reconcile_helper.go +++ b/internal/envfactory/reconcile_helper.go @@ -3,6 +3,7 @@ package envfactory import ( "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/internal/util" ) // BuildAcceptedSet converts an EnvironmentParams into the AcceptedSet and anchor @@ -27,19 +28,12 @@ import ( // force a fresh put. This is the single home for the anchor invariant. func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config.SDKKey, error) { anchor := params.SDKKey + b := credential.NewAcceptedSetBuilder().WithEnvironmentID(params.EnvID) - // WithAnchor / WithPrimaryMobileKey each add the key and designate it (the anchor and the - // wire's mobKey, respectively). An undefined key makes the call a no-op, so an undefined anchor - // leaves the set with no designated anchor and Build returns a *MalformedCredentialSetError. - b := credential.NewAcceptedSetBuilder(). - WithEnvironmentID(params.EnvID). - WithAnchor(anchor). - WithPrimaryMobileKey(params.MobileKey) - - // Validate and add the remaining accepted keys. The builder de-duplicates by value, so the - // anchor and the primary mobile key — already added permanently above — are ignored when they - // reappear in their arrays. That also defends the anchor-never-expiring invariant: a payload - // that (wrongly) carries an expiry on the anchor's own entry cannot demote it. + // Add every accepted SDK key, designating the anchor as we encounter it. WithAnchor both adds and + // designates, and forces the anchor permanent — so a payload that (wrongly) carries an expiry on + // the anchor's own entry cannot demote it. An undefined anchor never matches a (defined) array + // value, so it is never designated and Build returns a *MalformedCredentialSetError. // // Entries with an empty value are structurally malformed: relay would silently accept them but // they can never authenticate any SDK. Reject loudly rather than produce a credential-short env. @@ -50,22 +44,22 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. } if k.Value == anchor { anchorInArray = true - } - if k.Expiry.IsZero() { - b.WithSDKKey(k.Value) + b.WithAnchor(credential.SDKKeyParams{Value: k.Value, Key: util.PtrOrNil(k.Key)}) } else { - b.WithExpiringSDKKey(k.Value, k.Expiry) + b.WithSDKKey(credential.SDKKeyParams{Value: k.Value, Key: util.PtrOrNil(k.Key), Expiry: util.PtrOrNil(k.Expiry)}) } } // The anchor must be one of the accepted SDK keys: the backend lists it in sdkKeys[] (and ToParams // synthesizes it into the array for old-format payloads). A defined anchor absent from the array is - // a structurally malformed payload — reject it per §9 rather than letting WithAnchor above - // silently synthesize it into the set. + // a structurally malformed payload — reject it. if anchor.Defined() && !anchorInArray { return credential.AcceptedSet{}, anchor, credential.NewAnchorNotInSetError() } + // Add every accepted mobile key, designating the primary as we encounter it. Like the anchor, + // WithPrimaryMobileKey forces the primary permanent, so an expiry the payload may carry on the + // primary's own entry cannot demote it. primaryMobileInArray := false for _, k := range params.AcceptedMobileKeys { if !k.Value.Defined() { @@ -73,11 +67,9 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. } if k.Value == params.MobileKey { primaryMobileInArray = true - } - if k.Expiry.IsZero() { - b.WithMobileKey(k.Value) + b.WithPrimaryMobileKey(credential.MobileKeyParams{Value: k.Value, Key: util.PtrOrNil(k.Key)}) } else { - b.WithExpiringMobileKey(k.Value, k.Expiry) + b.WithMobileKey(credential.MobileKeyParams{Value: k.Value, Key: util.PtrOrNil(k.Key), Expiry: util.PtrOrNil(k.Expiry)}) } } diff --git a/internal/envfactory/reconcile_helper_test.go b/internal/envfactory/reconcile_helper_test.go index b3abff11..97479490 100644 --- a/internal/envfactory/reconcile_helper_test.go +++ b/internal/envfactory/reconcile_helper_test.go @@ -7,6 +7,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/internal/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -56,8 +57,8 @@ func TestBuildAcceptedSet_HappyPath(t *testing.T) { expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithAnchor("sdk-anchor"). - WithPrimaryMobileKey("mob-primary")) + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) // mobile has no identifier in makeParams fixture assert.Equal(t, expected, set) } @@ -80,17 +81,17 @@ func TestBuildAcceptedSet_MultipleKeys(t *testing.T) { expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithAnchor("sdk-anchor"). - WithSDKKey("sdk-service-a"). - WithExpiringSDKKey("sdk-old", expiry1). - WithPrimaryMobileKey("mob-primary")) + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). + WithSDKKey(credential.SDKKeyParams{Value: "sdk-service-a", Key: util.PtrOrNil("service-a")}). + WithSDKKey(credential.SDKKeyParams{Value: "sdk-old", Key: util.PtrOrNil("old-key"), Expiry: util.PtrOrNil(expiry1)}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) assert.Equal(t, expected, set) } -// TestBuildAcceptedSet_Rename verifies that a rename — same credential value, different key -// identifier — is a no-op: the returned AcceptedSet is identical regardless of the identifier. +// TestBuildAcceptedSet_Rename verifies that a rename — same credential value, different identifier +// — updates only the identifier in the AcceptedSet, not the accepted credential itself. The sets +// produced before and after a rename carry the same credentials but different identifier maps. func TestBuildAcceptedSet_Rename(t *testing.T) { - // Build AcceptedSet for the "before" and "after" of a rename. paramsOldName := makeParams( "sdk-anchor", []AcceptedSDKKey{{Key: "old-name", Value: "sdk-anchor"}}, @@ -107,7 +108,19 @@ func TestBuildAcceptedSet_Rename(t *testing.T) { require.NoError(t, errOld) require.NoError(t, errNew) - assert.Equal(t, setOld, setNew, "rename (same value, different key identifier) should produce the same AcceptedSet") + // The credential content is the same — only the identifier differs. + // When Reconcile applies the new set the display name is refreshed but no credential is added or removed. + assert.NotEqual(t, setOld, setNew, "rename changes the identifier map, so the AcceptedSets differ") + expectedOld := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("old-name")}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) + assert.Equal(t, expectedOld, setOld) + expectedNew := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("new-name")}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) + assert.Equal(t, expectedNew, setNew) } // TestBuildAcceptedSet_Deexpiry verifies that removing the expiry from an existing key (a @@ -143,23 +156,19 @@ func TestBuildAcceptedSet_Deexpiry(t *testing.T) { // The set built without expiry must include sdk-old as a permanent key. expectedPermanent := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithAnchor("sdk-anchor"). - WithSDKKey("sdk-old"). // permanent, no expiry - WithPrimaryMobileKey("mob-primary")) + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). + WithSDKKey(credential.SDKKeyParams{Value: "sdk-old", Key: util.PtrOrNil("old-key")}). // permanent, no expiry + WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) assert.Equal(t, expectedPermanent, setNoExpiry) // Sanity: the expiring and non-expiring versions are different. assert.NotEqual(t, setWithExpiry, setNoExpiry) } -// TestBuildAcceptedSet_AnchorNotInArray verifies that an anchor absent from AcceptedSDKKeys is no -// longer rejected here: WithAnchor adds and designates the anchor regardless, so the -// resulting set contains both the anchor and the array entry. Structural validation of the wire -// payload (anchor-absent-from-array) happens upstream when the payload is parsed into params. // TestBuildAcceptedSet_AnchorNotInArray verifies that a defined anchor absent from the sdkKeys[] array -// yields a *credential.MalformedCredentialSetError per design §9: the payload is structurally -// inconsistent (the designated primary is not in the authoritative array), so it must be rejected -// rather than silently synthesized into the set. +// yields a *credential.MalformedCredentialSetError: the payload is structurally inconsistent (the +// designated anchor is not in the authoritative array), so it must be rejected rather than silently +// synthesized into the set. func TestBuildAcceptedSet_AnchorNotInArray(t *testing.T) { params := makeParams( "sdk-anchor", @@ -213,7 +222,7 @@ func TestBuildAcceptedSet_NoMobileKey(t *testing.T) { expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithAnchor("sdk-anchor")) + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor"})) assert.Equal(t, expected, set) } @@ -273,10 +282,10 @@ func TestBuildAcceptedSet_MixedUpdate(t *testing.T) { expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithAnchor("sdk-new-anchor"). - WithSDKKey("sdk-b"). - WithSDKKey("sdk-c"). - WithPrimaryMobileKey("mob-primary")) + WithAnchor(credential.SDKKeyParams{Value: "sdk-new-anchor", Key: util.PtrOrNil("new-default")}). + WithSDKKey(credential.SDKKeyParams{Value: "sdk-b", Key: util.PtrOrNil("service-b")}). + WithSDKKey(credential.SDKKeyParams{Value: "sdk-c", Key: util.PtrOrNil("service-c")}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) assert.Equal(t, expected, set) } @@ -300,9 +309,9 @@ func TestBuildAcceptedSet_AnchorNeverExpiring(t *testing.T) { // Anchor is permanent (WithAnchor), not expiring — identical to a payload with no anchor expiry. expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithAnchor("sdk-anchor"). - WithSDKKey("sdk-service-a"). - WithPrimaryMobileKey("mob-primary")) + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). + WithSDKKey(credential.SDKKeyParams{Value: "sdk-service-a", Key: util.PtrOrNil("service-a")}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) assert.Equal(t, expected, set) } @@ -325,10 +334,9 @@ func TestBuildAcceptedSet_MultipleMobileKeys(t *testing.T) { require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithAnchor("sdk-anchor"). - WithMobileKey("mob-primary"). - WithMobileKey("mob-secondary"). - WithPrimaryMobileKey("mob-primary")) + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). + WithMobileKey(credential.MobileKeyParams{Value: "mob-secondary", Key: util.PtrOrNil("mob-2")}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary", Key: util.PtrOrNil("mob-1")})) assert.Equal(t, expected, set) } @@ -352,10 +360,9 @@ func TestBuildAcceptedSet_ExpiringMobileKey(t *testing.T) { require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithAnchor("sdk-anchor"). - WithMobileKey("mob-primary"). - WithExpiringMobileKey("mob-old", expiry1). - WithPrimaryMobileKey("mob-primary")) + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). + WithMobileKey(credential.MobileKeyParams{Value: "mob-old", Key: util.PtrOrNil("mob-old"), Expiry: util.PtrOrNil(expiry1)}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary", Key: util.PtrOrNil("mob-1")})) assert.Equal(t, expected, set, "expiring mobile key must land as an expiring key in the set") } @@ -382,7 +389,7 @@ func TestBuildAcceptedSet_TrustTheArray(t *testing.T) { require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). - WithAnchor("sdk-anchor"). - WithPrimaryMobileKey("mob-primary")) + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) assert.Equal(t, expected, set, "legacy sdkKey.expiring slot must not appear in AcceptedSet") } diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index 654e8d9e..e77b7eaa 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -203,7 +203,7 @@ func TestAddRemoveCredential(t *testing.T) { // Reconcile to the full set: the SDK key (anchor) plus a mobile key and an environment ID. env.ReconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(envConfig.SDKKey).WithPrimaryMobileKey(mobileKey).WithEnvironmentID(envID))) + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}).WithPrimaryMobileKey(credential.MobileKeyParams{Value: mobileKey}).WithEnvironmentID(envID))) creds := env.GetCredentials() assert.Len(t, creds, 3) @@ -214,7 +214,7 @@ func TestAddRemoveCredential(t *testing.T) { // Reconciling with a different mobile key evicts the previous one. newMobileKey := config.MobileKey("evict-the-previous-key") env.ReconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(envConfig.SDKKey).WithPrimaryMobileKey(newMobileKey).WithEnvironmentID(envID))) + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}).WithPrimaryMobileKey(credential.MobileKeyParams{Value: newMobileKey}).WithEnvironmentID(envID))) creds = env.GetCredentials() assert.Len(t, creds, 3) @@ -236,7 +236,7 @@ func TestAddExistingCredentialDoesNothing(t *testing.T) { assert.Equal(t, []credential.SDKCredential{envConfig.SDKKey}, env.GetCredentials()) mobileKey := st.EnvWithAllCredentials.Config.MobileKey - set := mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(envConfig.SDKKey).WithPrimaryMobileKey(mobileKey)) + set := mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}).WithPrimaryMobileKey(credential.MobileKeyParams{Value: mobileKey})) env.ReconcileCredentials(set) @@ -286,8 +286,8 @@ func TestChangeSDKKey(t *testing.T) { // Upon rotating to key2, the original key should still be valid for an hour. rotationSet, err := credential.NewAcceptedSetBuilder(). - WithAnchor(key2). - WithExpiringSDKKey(envConfig.SDKKey, start.Add(1*time.Hour)). + WithAnchor(credential.SDKKeyParams{Value: key2}). + WithSDKKey(credential.SDKKeyParams{Value: envConfig.SDKKey, Expiry: util.PtrOrNil(start.Add(1 * time.Hour))}). Build() require.NoError(t, err) envImpl.reconcileCredentials(rotationSet, start) @@ -358,9 +358,9 @@ func TestMobileKeyReconcileExpiry(t *testing.T) { // carries a per-key expiry. envImpl.reconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithAnchor(envConfig.SDKKey). - WithPrimaryMobileKey(primaryMobile). - WithExpiringMobileKey(expiringMobile, expiry)), + WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: primaryMobile}). + WithMobileKey(credential.MobileKeyParams{Value: expiringMobile, Expiry: util.PtrOrNil(expiry)})), start) // Reconcile stores the expiry as data, so before it elapses the key is accepted (not deprecated). @@ -403,9 +403,9 @@ func TestNonAnchorSDKKeysDoNotOpenUpstreamClient(t *testing.T) { // open an upstream client. env.ReconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithAnchor(envConfig.SDKKey). - WithSDKKey(nonAnchorKey1). - WithSDKKey(nonAnchorKey2))) + WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). + WithSDKKey(credential.SDKKeyParams{Value: nonAnchorKey1}). + WithSDKKey(credential.SDKKeyParams{Value: nonAnchorKey2}))) // All three SDK keys are accepted... creds := env.GetCredentials() @@ -448,9 +448,9 @@ func TestGetClientReturnsAnchorInMultiKeyEnv(t *testing.T) { env.ReconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithAnchor(envConfig.SDKKey). - WithSDKKey(nonAnchorKey1). - WithSDKKey(nonAnchorKey2))) + WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). + WithSDKKey(credential.SDKKeyParams{Value: nonAnchorKey1}). + WithSDKKey(credential.SDKKeyParams{Value: nonAnchorKey2}))) // No new upstream client was created for the non-anchor keys. if !helpers.AssertNoMoreValues(t, clientCh, 200*time.Millisecond) { @@ -491,9 +491,9 @@ func TestNonPrimaryMobileKeyDoesNotStealEventForwarding(t *testing.T) { // non-primary mobile key. Accepting the non-primary key must NOT repoint event forwarding — // events collapse to the primary mobile key, mirroring the SDK anchor. env.ReconcileCredentials(mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithAnchor(envConfig.SDKKey). - WithPrimaryMobileKey(primaryMobile). - WithMobileKey(nonPrimaryMobile). + WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: primaryMobile}). + WithMobileKey(credential.MobileKeyParams{Value: nonPrimaryMobile}). WithEnvironmentID(envConfig.EnvID))) ed := envImpl.GetEventDispatcher() @@ -551,8 +551,8 @@ func TestReAnchoringToKeyStillInGraceReusesItsClient(t *testing.T) { // alive because keyA is still accepted during the grace window. env.(*envContextImpl).reconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithAnchor(keyB). - WithExpiringSDKKey(keyA, start.Add(1*time.Hour))), + WithAnchor(credential.SDKKeyParams{Value: keyB}). + WithSDKKey(credential.SDKKeyParams{Value: keyA, Expiry: util.PtrOrNil(start.Add(1 * time.Hour))})), start) clientB := requireClientReady(t, clientCh) @@ -566,7 +566,7 @@ func TestReAnchoringToKeyStillInGraceReusesItsClient(t *testing.T) { // being started -- so there is no stale client to orphan. keyB is omitted from the set (no expiry), // so it is revoked immediately and its client is closed. env.(*envContextImpl).reconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(keyA)), + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(credential.SDKKeyParams{Value: keyA})), start.Add(10*time.Minute)) // keyB was revoked by the re-anchor, so its client is closed. @@ -641,7 +641,7 @@ func TestRevokingSDKKeyWhileClientIsStartingDoesNotLeakTheClient(t *testing.T) { // runs now -- but c.clients[keyA] is still nil because the initial goroutine is blocked in the factory, // so nothing is closed and the mapping is simply removed. env.(*envContextImpl).reconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(keyB)), + mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(credential.SDKKeyParams{Value: keyB})), time.Unix(1000, 0)) creds := env.GetCredentials() diff --git a/internal/relayenv/env_context_reanchor_test.go b/internal/relayenv/env_context_reanchor_test.go index ee6e8c2a..d967cf96 100644 --- a/internal/relayenv/env_context_reanchor_test.go +++ b/internal/relayenv/env_context_reanchor_test.go @@ -24,14 +24,15 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/basictypes" - "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/bigsegments" + "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/httpconfig" "github.com/launchdarkly/ld-relay/v8/internal/sdks" st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" "github.com/launchdarkly/ld-relay/v8/internal/store" "github.com/launchdarkly/ld-relay/v8/internal/streams" + "github.com/launchdarkly/ld-relay/v8/internal/util" "github.com/launchdarkly/eventsource" "github.com/launchdarkly/go-sdk-common/v3/ldlog" @@ -103,8 +104,8 @@ func (f *sharedStoreFactory) Build(_ subsystems.ClientContext) (subsystems.DataS func reanchor(t *testing.T, env EnvContext, newKey, oldKey config.SDKKey, now time.Time) { t.Helper() set, err := credential.NewAcceptedSetBuilder(). - WithAnchor(newKey). - WithExpiringSDKKey(oldKey, now.Add(time.Hour)). + WithAnchor(credential.SDKKeyParams{Value: newKey}). + WithSDKKey(credential.SDKKeyParams{Value: oldKey, Expiry: util.PtrOrNil(now.Add(time.Hour))}). Build() require.NoError(t, err) env.(*envContextImpl).reconcileCredentials(set, now) diff --git a/internal/util/pointer.go b/internal/util/pointer.go new file mode 100644 index 00000000..1e9425f8 --- /dev/null +++ b/internal/util/pointer.go @@ -0,0 +1,11 @@ +package util + +// PtrOrNil returns a pointer to v, or nil when v is the zero value for its type. It is used to model +// optional fields where the zero value (e.g. an empty string or the zero time.Time) means "absent". +func PtrOrNil[T comparable](v T) *T { + var zero T + if v == zero { + return nil + } + return &v +} diff --git a/internal/util/pointer_test.go b/internal/util/pointer_test.go new file mode 100644 index 00000000..49a56743 --- /dev/null +++ b/internal/util/pointer_test.go @@ -0,0 +1,32 @@ +package util + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPtrOrNil(t *testing.T) { + t.Run("empty string yields nil", func(t *testing.T) { + assert.Nil(t, PtrOrNil("")) + }) + + t.Run("non-empty string yields pointer to value", func(t *testing.T) { + p := PtrOrNil("default") + require.NotNil(t, p) + assert.Equal(t, "default", *p) + }) + + t.Run("zero time yields nil", func(t *testing.T) { + assert.Nil(t, PtrOrNil(time.Time{})) + }) + + t.Run("non-zero time yields pointer to value", func(t *testing.T) { + now := time.Unix(1000, 0) + p := PtrOrNil(now) + require.NotNil(t, p) + assert.Equal(t, now, *p) + }) +} From 46f7376464bcf5d686f880eea961de88f3287b1c Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 30 Jun 2026 13:25:57 -0700 Subject: [PATCH 29/66] feat(credential): expose the full accepted key set via AcceptedKeys (#730) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose the full accepted credential set via Rotator.AcceptedKeys() and EnvContext.GetAcceptedKeys() — every server-side SDK key and mobile key, including the anchor and primary mobile key — so the status endpoint can surface it. --- internal/credential/accepted_set.go | 6 +- internal/credential/accepted_set_builder.go | 12 +-- internal/credential/rotator.go | 87 +++++++++++++++------ internal/credential/rotator_test.go | 87 ++++++++++++++++++++- internal/relayenv/env_context.go | 6 ++ internal/relayenv/env_context_impl.go | 4 + 6 files changed, 167 insertions(+), 35 deletions(-) diff --git a/internal/credential/accepted_set.go b/internal/credential/accepted_set.go index 9366ae30..dc65c3c6 100644 --- a/internal/credential/accepted_set.go +++ b/internal/credential/accepted_set.go @@ -28,10 +28,10 @@ import ( type AcceptedSet struct { // sdkKeys and mobileKeys store each accepted key once, keyed by value (the secret), so duplicates // collapse without a containment scan. The map value carries the key's metadata (see - // acceptedKeyInfo). A nil map is a valid empty set (reads return absent; only the builder writes). - sdkKeys map[config.SDKKey]acceptedKeyInfo + // AcceptedKey). A nil map is a valid empty set (reads return absent; only the builder writes). + sdkKeys map[config.SDKKey]AcceptedKey anchor config.SDKKey - mobileKeys map[config.MobileKey]acceptedKeyInfo + mobileKeys map[config.MobileKey]AcceptedKey primaryMobileKey config.MobileKey envID config.EnvironmentID } diff --git a/internal/credential/accepted_set_builder.go b/internal/credential/accepted_set_builder.go index d3b4b561..47136a8b 100644 --- a/internal/credential/accepted_set_builder.go +++ b/internal/credential/accepted_set_builder.go @@ -16,8 +16,8 @@ type AcceptedSetBuilder struct { func NewAcceptedSetBuilder() *AcceptedSetBuilder { return &AcceptedSetBuilder{ set: AcceptedSet{ - sdkKeys: make(map[config.SDKKey]acceptedKeyInfo), - mobileKeys: make(map[config.MobileKey]acceptedKeyInfo), + sdkKeys: make(map[config.SDKKey]AcceptedKey), + mobileKeys: make(map[config.MobileKey]AcceptedKey), }, } } @@ -43,7 +43,7 @@ func (b *AcceptedSetBuilder) WithSDKKey(p SDKKeyParams) *AcceptedSetBuilder { if !p.Value.Defined() || b.set.hasSDKKey(p.Value) { return b } - b.set.sdkKeys[p.Value] = acceptedKeyInfo{key: p.Key, expiry: p.Expiry} + b.set.sdkKeys[p.Value] = AcceptedKey{Key: p.Key, Expiry: p.Expiry} return b } @@ -55,7 +55,7 @@ func (b *AcceptedSetBuilder) WithAnchor(p SDKKeyParams) *AcceptedSetBuilder { if !p.Value.Defined() { return b } - b.set.sdkKeys[p.Value] = acceptedKeyInfo{key: p.Key, expiry: nil} + b.set.sdkKeys[p.Value] = AcceptedKey{Key: p.Key, Expiry: nil} b.set.anchor = p.Value return b } @@ -65,7 +65,7 @@ func (b *AcceptedSetBuilder) WithMobileKey(p MobileKeyParams) *AcceptedSetBuilde if !p.Value.Defined() || b.set.hasMobileKey(p.Value) { return b } - b.set.mobileKeys[p.Value] = acceptedKeyInfo{key: p.Key, expiry: p.Expiry} + b.set.mobileKeys[p.Value] = AcceptedKey{Key: p.Key, Expiry: p.Expiry} return b } @@ -76,7 +76,7 @@ func (b *AcceptedSetBuilder) WithPrimaryMobileKey(p MobileKeyParams) *AcceptedSe if !p.Value.Defined() { return b } - b.set.mobileKeys[p.Value] = acceptedKeyInfo{key: p.Key, expiry: nil} + b.set.mobileKeys[p.Value] = AcceptedKey{Key: p.Key, Expiry: nil} b.set.primaryMobileKey = p.Value return b } diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 887e064d..db6881da 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -1,6 +1,7 @@ package credential import ( + "maps" "sync" "time" @@ -8,10 +9,29 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" ) -// acceptedKeyInfo holds per-key metadata for the accepted-set maps. -type acceptedKeyInfo struct { - expiry *time.Time // nil = permanent - key *string // wire "key" identifier — non-secret human-readable name; nil when absent +// AcceptedKey is the metadata for one accepted credential: its optional expiry and optional wire +// "key" identifier. The credential value itself is the map key wherever AcceptedKey is stored — the +// rotator's accepted-key maps, the builder's AcceptedSet, and the AcceptedKeySet returned by +// AcceptedKeys. +type AcceptedKey struct { + // Expiry is the key's expiry. A nil expiry means the key is permanent. + Expiry *time.Time + // Key is the non-secret wire "key" identifier — a human-readable name. Nil when the source carried + // none (manual configuration, or an old-format payload predating concurrent keys). + Key *string +} + +// AcceptedKeySet is a point-in-time snapshot of an environment's full accepted credential set, +// returned by Rotator.AcceptedKeys. Server and Mobile are keyed by credential value (the secret); +// the value AcceptedKey carries that key's metadata. Anchor and PrimaryMobile name the designated +// keys within Server and Mobile. The status endpoint maps Server/Mobile to the sdkKeys[]/mobileKeys[] +// arrays and uses Anchor to mark the anchor entry. Reads of the maps and the designations are taken +// under a single lock, so they are mutually consistent. +type AcceptedKeySet struct { + Server map[config.SDKKey]AcceptedKey + Mobile map[config.MobileKey]AcceptedKey + Anchor config.SDKKey + PrimaryMobile config.MobileKey } type Rotator struct { @@ -29,11 +49,11 @@ type Rotator struct { // acceptedSDKKeys is the full set of accepted SDK keys with optional per-key expiry. // A nil expiry means the key is permanent. The anchor is always present with a nil expiry. - acceptedSDKKeys map[config.SDKKey]acceptedKeyInfo + acceptedSDKKeys map[config.SDKKey]AcceptedKey // acceptedMobileKeys is the full set of accepted mobile keys with optional per-key expiry. // A nil expiry means the key is permanent. - acceptedMobileKeys map[config.MobileKey]acceptedKeyInfo + acceptedMobileKeys map[config.MobileKey]AcceptedKey expirations []SDKCredential additions []SDKCredential @@ -52,8 +72,8 @@ type InitialCredentials struct { func NewRotator(loggers ldlog.Loggers) *Rotator { r := &Rotator{ loggers: loggers, - acceptedSDKKeys: make(map[config.SDKKey]acceptedKeyInfo), - acceptedMobileKeys: make(map[config.MobileKey]acceptedKeyInfo), + acceptedSDKKeys: make(map[config.SDKKey]AcceptedKey), + acceptedMobileKeys: make(map[config.MobileKey]AcceptedKey), } return r } @@ -71,10 +91,10 @@ func (r *Rotator) Initialize(credentials []SDKCredential) { switch cred := cred.(type) { case config.SDKKey: r.anchorKey = cred - r.acceptedSDKKeys[cred] = acceptedKeyInfo{} + r.acceptedSDKKeys[cred] = AcceptedKey{} case config.MobileKey: r.primaryMobileKey = cred - r.acceptedMobileKeys[cred] = acceptedKeyInfo{} + r.acceptedMobileKeys[cred] = AcceptedKey{} case config.EnvironmentID: r.primaryEnvironmentID = cred } @@ -120,19 +140,17 @@ func (r *Rotator) allCredentials() []SDKCredential { // DeprecatedCredentials returns the SDK keys being phased out — every accepted SDK key, other than the // anchor, that carries a future expiry. (Per-key expiry is stored as data on the accepted entry; the -// cleanup ticker drops the key once it elapses.) EnvContext.GetDeprecatedCredentials delegates here to -// populate the status endpoint's expiringSdkKey field. +// cleanup ticker drops the key once it elapses.) // // Mobile keys are deliberately not returned even though they expire the same way SDK keys do — carried -// as per-key expiry and dropped by the same cleanup ticker. They are omitted only because the status -// endpoint has no expiringMobileKey field to populate, not because mobile-key expiry is unimplemented. +// as per-key expiry and dropped by the same cleanup ticker. func (r *Rotator) DeprecatedCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() var out []SDKCredential for key, info := range r.acceptedSDKKeys { - if info.expiry != nil && key != r.anchorKey { + if info.Expiry != nil && key != r.anchorKey { out = append(out, key) } } @@ -167,7 +185,7 @@ func (r *Rotator) expireMobileKey(mobileKey config.MobileKey) { // expirations for the tracked credentials since the last time this method was called. // // It enforces per-key expiry for both SDK and mobile keys: expiry is stored as data on the accepted -// entry (acceptedKeyInfo.expiry); a nil expiry means the key is permanent and is never expired here. +// entry (AcceptedKey.Expiry); a nil expiry means the key is permanent and is never expired here. // // Expiry happens strictly after a key's expiry timestamp. func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expirations []SDKCredential) { @@ -175,12 +193,12 @@ func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expiration defer r.mu.Unlock() for key, info := range r.acceptedSDKKeys { - if info.expiry != nil && now.After(*info.expiry) { + if info.Expiry != nil && now.After(*info.Expiry) { r.expireSDKKey(key) } } for key, info := range r.acceptedMobileKeys { - if info.expiry != nil && now.After(*info.expiry) { + if info.Expiry != nil && now.After(*info.Expiry) { r.expireMobileKey(key) } } @@ -223,8 +241,8 @@ type reconcilableKey interface { // key no longer desired is dropped and queued as an expiration. Per-key expiry is stored as data on // the accepted entry; the cleanup ticker is what later acts on it. The caller must hold the write lock. func reconcileAcceptedKeys[K reconcilableKey]( - desired map[K]acceptedKeyInfo, - accepted map[K]acceptedKeyInfo, + desired map[K]AcceptedKey, + accepted map[K]AcceptedKey, additions *[]SDKCredential, expirations *[]SDKCredential, loggers ldlog.Loggers, @@ -259,9 +277,9 @@ func reconcileAcceptedKeys[K reconcilableKey]( // the anchor is present and permanent (WithAnchor forces a nil expiry), so no special handling is // needed here. The caller must hold the write lock. func (r *Rotator) reconcileSDKKeys(set AcceptedSet, anchor config.SDKKey, now time.Time) { - desired := make(map[config.SDKKey]acceptedKeyInfo, len(set.sdkKeys)) + desired := make(map[config.SDKKey]AcceptedKey, len(set.sdkKeys)) for key, info := range set.sdkKeys { - if info.expiry != nil && !now.Before(*info.expiry) { + if info.Expiry != nil && !now.Before(*info.Expiry) { continue // already expired; treat as absent } desired[key] = info @@ -275,9 +293,9 @@ func (r *Rotator) reconcileSDKKeys(set AcceptedSet, anchor config.SDKKey, now ti // (WithPrimaryMobileKey forces a nil expiry). An empty primary means the set declared no mobile key. // The caller must hold the lock. func (r *Rotator) reconcileMobileKeys(set AcceptedSet, now time.Time) { - desired := make(map[config.MobileKey]acceptedKeyInfo, len(set.mobileKeys)) + desired := make(map[config.MobileKey]AcceptedKey, len(set.mobileKeys)) for key, info := range set.mobileKeys { - if info.expiry != nil && !now.Before(*info.expiry) { + if info.Expiry != nil && !now.Before(*info.Expiry) { continue // already expired; treat as absent } desired[key] = info @@ -298,3 +316,24 @@ func (r *Rotator) reconcileEnvironmentID(set AcceptedSet) { r.primaryEnvironmentID = set.envID r.additions = append(r.additions, set.envID) } + +// AcceptedKeys returns a snapshot of the full accepted credential set — all server-side SDK keys and +// all mobile keys (anchor and primary mobile key included) — grouped by kind, along with which keys +// are the designated anchor and primary mobile. The maps and the designations are read under a single +// lock so they are mutually consistent. The status endpoint maps each group to the sdkKeys[] / +// mobileKeys[] arrays. +func (r *Rotator) AcceptedKeys() AcceptedKeySet { + r.mu.RLock() + defer r.mu.RUnlock() + + server := make(map[config.SDKKey]AcceptedKey, len(r.acceptedSDKKeys)) + maps.Copy(server, r.acceptedSDKKeys) + mobile := make(map[config.MobileKey]AcceptedKey, len(r.acceptedMobileKeys)) + maps.Copy(mobile, r.acceptedMobileKeys) + return AcceptedKeySet{ + Server: server, + Mobile: mobile, + Anchor: r.anchorKey, + PrimaryMobile: r.primaryMobileKey, + } +} diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 91238134..b89eb56a 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -34,13 +34,13 @@ func TestInitializePopulatesAcceptedSets(t *testing.T) { // Verify accepted SDK key set: one entry, no expiry. assert.Len(t, rotator.acceptedSDKKeys, 1) if info, ok := rotator.acceptedSDKKeys[sdkKey]; assert.True(t, ok, "acceptedSDKKeys should contain the initialized SDK key") { - assert.Nil(t, info.expiry, "a key initialized without expiry should have nil expiry in acceptedKeyInfo") + assert.Nil(t, info.Expiry, "a key initialized without expiry should have nil expiry in AcceptedKey") } // Verify accepted mobile key set: one entry, no expiry. assert.Len(t, rotator.acceptedMobileKeys, 1) if info, ok := rotator.acceptedMobileKeys[mobileKey]; assert.True(t, ok, "acceptedMobileKeys should contain the initialized mobile key") { - assert.Nil(t, info.expiry, "a key initialized without expiry should have nil expiry in acceptedKeyInfo") + assert.Nil(t, info.Expiry, "a key initialized without expiry should have nil expiry in AcceptedKey") } // Existing public API is unchanged. @@ -266,3 +266,86 @@ func TestReconcileDeExpiryRestoresKey(t *testing.T) { assert.Empty(t, expirations) assert.Contains(t, r.AllCredentials(), SDKCredential(key)) } + +// TestAcceptedKeys verifies that AcceptedKeys returns the full accepted set — every server and mobile +// key, including the anchor and primary mobile key — grouped by kind with identifier and expiry +// populated, plus the anchor and primary-mobile designations. +func TestAcceptedKeys(t *testing.T) { + t.Run("single anchor plus primary mobile", func(t *testing.T) { + r := newTestRotator() + r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). + WithPrimaryMobileKey(MobileKeyParams{Value: "mob-primary", Key: util.PtrOrNil("mob-1")})), time.Unix(0, 0)) + + set := r.AcceptedKeys() + require.Len(t, set.Server, 1) + require.Len(t, set.Mobile, 1) + assert.Equal(t, config.SDKKey("sdk-anchor"), set.Anchor) + assert.Equal(t, config.MobileKey("mob-primary"), set.PrimaryMobile) + + anchor, ok := set.Server["sdk-anchor"] + require.True(t, ok) + require.NotNil(t, anchor.Key) + assert.Equal(t, "default", *anchor.Key) + assert.Nil(t, anchor.Expiry) + + mob, ok := set.Mobile["mob-primary"] + require.True(t, ok) + require.NotNil(t, mob.Key) + assert.Equal(t, "mob-1", *mob.Key) + }) + + t.Run("multiple keys include the anchor; expiry populated", func(t *testing.T) { + r := newTestRotator() + expiry := time.Date(2099, 6, 1, 0, 0, 0, 0, time.UTC) + r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). + WithSDKKey(SDKKeyParams{Value: "sdk-b", Key: util.PtrOrNil("b-service")}). + WithSDKKey(SDKKeyParams{Value: "sdk-old", Key: util.PtrOrNil("old-key"), Expiry: util.PtrOrNil(expiry)}). + WithPrimaryMobileKey(MobileKeyParams{Value: "mob-primary"})), time.Unix(0, 0)) + + set := r.AcceptedKeys() + require.Len(t, set.Server, 3) // anchor + sdk-b + sdk-old + require.Len(t, set.Mobile, 1) + _, ok := set.Server["sdk-anchor"] + assert.True(t, ok, "anchor must be present in the full set") + + old, ok := set.Server["sdk-old"] + require.True(t, ok) + require.NotNil(t, old.Expiry) + assert.Equal(t, expiry, *old.Expiry) + + // A key with no identifier (the primary mobile here) carries a nil Key. + mob, ok := set.Mobile["mob-primary"] + require.True(t, ok) + assert.Nil(t, mob.Key) + }) +} + +// TestReconcileClearsStaleKeyIdentifier verifies that when a later reconcile carries no identifier for +// a key that previously had one (e.g. an old-format payload after a new-format one), the rotator +// clears the stale identifier rather than retaining it — so /status never shows an identifier the +// current credential set no longer carries. +func TestReconcileClearsStaleKeyIdentifier(t *testing.T) { + r := newTestRotator() + now := time.Unix(0, 0) + + // First reconcile: sdk-b carries the identifier "b-service". + r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: "sdk-anchor"}). + WithSDKKey(SDKKeyParams{Value: "sdk-b", Key: util.PtrOrNil("b-service")})), now) + + b, ok := r.AcceptedKeys().Server["sdk-b"] + require.True(t, ok) + require.NotNil(t, b.Key) + assert.Equal(t, "b-service", *b.Key) + + // Second reconcile: same credential value, but no identifier this time. + r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: "sdk-anchor"}). + WithSDKKey(SDKKeyParams{Value: "sdk-b"})), now) + + b, ok = r.AcceptedKeys().Server["sdk-b"] + require.True(t, ok) + assert.Nil(t, b.Key, "identifier must be cleared when the new payload carries none") +} diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index 123ae722..f9b5c9a1 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -50,6 +50,12 @@ type EnvContext interface { // several accepted mobile keys (primary + expiring) in nondeterministic order. GetMobileKey() config.MobileKey + // GetAcceptedKeys returns a consistent snapshot of the full accepted credential set — all + // server-side SDK keys and all mobile keys (anchor and primary mobile key included), grouped by + // kind, plus which keys are the designated anchor and primary. The status endpoint maps each group + // to the full sdkKeys[] / mobileKeys[] arrays. + GetAcceptedKeys() credential.AcceptedKeySet + // ReconcileCredentials atomically reconciles the environment's accepted credentials to match // newSet. The set names its own anchor (the SDK key that owns the upstream connection) and // primary mobile key. The method owns the order of operations internally (add → re-anchor → diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 964d9659..7bad18ae 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -640,6 +640,10 @@ func (c *envContextImpl) GetDeprecatedCredentials() []credential.SDKCredential { return c.keyRotator.DeprecatedCredentials() } +func (c *envContextImpl) GetAcceptedKeys() credential.AcceptedKeySet { + return c.keyRotator.AcceptedKeys() +} + func (c *envContextImpl) GetClient() sdks.LDClientContext { c.mu.RLock() defer c.mu.RUnlock() From 9737404915abcfb871e21fa41c4c3395e74f1315 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 30 Jun 2026 13:36:51 -0700 Subject: [PATCH 30/66] feat(status): surface full sdkKeys[]/mobileKeys[] arrays on /status (#731) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface the full accepted credential set on /status as sdkKeys[] / mobileKeys[] arrays — each entry carrying the obscured value, the optional wire identifier, and an optional Unix-millisecond expiry. The existing scalar sdkKey / mobileKey fields are preserved and now designate which array entry is the anchor / primary. --- internal/api/status_reps.go | 38 +++++++-- relay/endpoints_status.go | 73 ++++++++++++++--- relay/endpoints_status_test.go | 145 +++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 17 deletions(-) diff --git a/internal/api/status_reps.go b/internal/api/status_reps.go index 7618d16e..6d72298f 100644 --- a/internal/api/status_reps.go +++ b/internal/api/status_reps.go @@ -15,17 +15,41 @@ type StatusRep struct { ClientVersion string `json:"clientVersion"` } +// KeyStatus is the JSON representation of one accepted SDK or mobile key in the status endpoint's +// sdkKeys[] / mobileKeys[] arrays. +// +// Key is the non-secret human-readable identifier from the wire format (the "key" field of a +// sdkKeys/mobileKeys entry); it is omitted when the source carried no identifier (manual config, or +// an old-format payload predating concurrent keys). Value is the obscured credential secret (via +// sdks.ObscureKey). Expiry carries the Unix-millisecond expiry timestamp when the key is being phased +// out; it is omitted for permanent keys. +type KeyStatus struct { + Key string `json:"key,omitempty"` + Value string `json:"value"` + Expiry *int64 `json:"expiry,omitempty"` +} + // EnvironmentStatusRep is the per-environment JSON representation returned by the status endpoint. // // This is exported for use in integration test code. type EnvironmentStatusRep struct { - SDKKey string `json:"sdkKey"` - EnvID string `json:"envId,omitempty"` - EnvKey string `json:"envKey,omitempty"` - EnvName string `json:"envName,omitempty"` - ProjKey string `json:"projKey,omitempty"` - ProjName string `json:"projName,omitempty"` - MobileKey string `json:"mobileKey,omitempty"` + // SDKKey is the obscured anchor SDK key — the key relay uses for its upstream connection. It + // designates which SDKKeys entry is the anchor. + SDKKey string `json:"sdkKey"` + // SDKKeys carries the full accepted set of server-side SDK keys — including the anchor — with their + // identifiers, obscured values, and optional expiry. Always present; always contains at least the + // anchor. + SDKKeys []KeyStatus `json:"sdkKeys"` + EnvID string `json:"envId,omitempty"` + EnvKey string `json:"envKey,omitempty"` + EnvName string `json:"envName,omitempty"` + ProjKey string `json:"projKey,omitempty"` + ProjName string `json:"projName,omitempty"` + // MobileKey is the obscured primary mobile key. It designates which MobileKeys entry is the primary. + MobileKey string `json:"mobileKey,omitempty"` + // MobileKeys carries the full accepted set of mobile keys — including the primary. Always present; + // empty for an environment with no mobile keys (e.g. server-side only). + MobileKeys []KeyStatus `json:"mobileKeys"` ExpiringSDKKey string `json:"expiringSdkKey,omitempty"` Status string `json:"status"` ConnectionStatus ConnectionStatusRep `json:"connectionStatus"` diff --git a/relay/endpoints_status.go b/relay/endpoints_status.go index cce75869..725cf26a 100644 --- a/relay/endpoints_status.go +++ b/relay/endpoints_status.go @@ -3,10 +3,13 @@ package relay import ( "encoding/json" "net/http" + "slices" + "strings" "time" "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/api" + "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/relayenv" "github.com/launchdarkly/ld-relay/v8/internal/sdks" @@ -46,25 +49,54 @@ func statusHandler(relay *Relay) http.Handler { ProjName: identifiers.ProjName, } - // Use the anchor SDK key and primary mobile key specifically — GetCredentials() may return - // multiple SDK and mobile keys (primary + expiring), so iterating it for these singular - // status fields would give a non-deterministic result. - if key := clientCtx.GetAnchorKey(); key.Defined() { - status.SDKKey = sdks.ObscureKey(string(key)) + // One consistent snapshot of the accepted credential set drives every credential field + // below — the scalar anchor/primary designations, the full sdkKeys[]/mobileKeys[] arrays, + // and expiringSdkKey — so they cannot drift relative to each other under a concurrent + // reconcile. + accepted := clientCtx.GetAcceptedKeys() + + // Scalar fields: the anchor SDK key and primary mobile key designate which array entry is + // the anchor / primary. + if accepted.Anchor.Defined() { + status.SDKKey = sdks.ObscureKey(string(accepted.Anchor)) } - if key := clientCtx.GetMobileKey(); key.Defined() { - status.MobileKey = sdks.ObscureKey(string(key)) + if accepted.PrimaryMobile.Defined() { + status.MobileKey = sdks.ObscureKey(string(accepted.PrimaryMobile)) } for _, c := range clientCtx.GetCredentials() { if envID, ok := c.(config.EnvironmentID); ok { status.EnvID = string(envID) } } - for _, c := range clientCtx.GetDeprecatedCredentials() { - if key, ok := c.(config.SDKKey); ok { - status.ExpiringSDKKey = sdks.ObscureKey(string(key)) + + // sdkKeys[] / mobileKeys[]: the full accepted set, grouped by kind. Always present (never + // null): a server-only env has an empty mobileKeys. Order is unspecified. + status.SDKKeys = make([]api.KeyStatus, 0, len(accepted.Server)) + var expiringCandidates []expiringSDKKey + for value, info := range accepted.Server { + status.SDKKeys = append(status.SDKKeys, keyStatus(string(value), info)) + // expiringSdkKey considers non-anchor server keys that carry an expiry. + if value != accepted.Anchor && info.Expiry != nil { + expiringCandidates = append(expiringCandidates, expiringSDKKey{value: string(value), expiry: *info.Expiry}) } } + status.MobileKeys = make([]api.KeyStatus, 0, len(accepted.Mobile)) + for value, info := range accepted.Mobile { + status.MobileKeys = append(status.MobileKeys, keyStatus(string(value), info)) + } + + // expiringSdkKey: the soonest-expiring non-anchor SDK key. Comparing by expiry then by value + // gives a total order, so the chosen key is deterministic even when several keys share the + // same expiry (map iteration order, and hence MinFunc's pick on a tie, is otherwise unstable). + if len(expiringCandidates) > 0 { + earliest := slices.MinFunc(expiringCandidates, func(a, b expiringSDKKey) int { + if c := a.expiry.Compare(b.expiry); c != 0 { + return c + } + return strings.Compare(a.value, b.value) + }) + status.ExpiringSDKKey = sdks.ObscureKey(earliest.value) + } client := clientCtx.GetClient() if client == nil { @@ -155,3 +187,24 @@ func statusHandler(relay *Relay) http.Handler { _, _ = w.Write(data) }) } + +// expiringSDKKey is a candidate for the status endpoint's expiringSdkKey field: a non-anchor SDK key +// that carries an expiry. value is the plain credential; expiry is its (non-nil) expiry. +type expiringSDKKey struct { + value string + expiry time.Time +} + +// keyStatus converts an accepted key — its credential value plus metadata — into its status-endpoint +// JSON representation, obscuring the secret value and surfacing the optional identifier and expiry. +func keyStatus(value string, k credential.AcceptedKey) api.KeyStatus { + ks := api.KeyStatus{Value: sdks.ObscureKey(value)} + if k.Key != nil { + ks.Key = *k.Key + } + if k.Expiry != nil { + ms := k.Expiry.UnixMilli() + ks.Expiry = &ms + } + return ks +} diff --git a/relay/endpoints_status_test.go b/relay/endpoints_status_test.go index ffbcdad7..9527e4cb 100644 --- a/relay/endpoints_status_test.go +++ b/relay/endpoints_status_test.go @@ -5,12 +5,14 @@ import ( "testing" "time" + "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" c "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/sdks" st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + "github.com/launchdarkly/ld-relay/v8/internal/util" ct "github.com/launchdarkly/go-configtypes" "github.com/launchdarkly/go-sdk-common/v3/ldtime" @@ -56,6 +58,36 @@ func TestEndpointsStatus(t *testing.T) { st.AssertJSONPathMatch(t, p.relay.version, status, "version") st.AssertJSONPathMatch(t, ld.Version, status, "clientVersion") }) + + t.Run("sdkKeys/mobileKeys arrays carry the full accepted set including the anchor", func(t *testing.T) { + var config c.Config + config.Environment = st.MakeEnvConfigs(st.EnvMain, st.EnvMobile) + + withStartedRelay(t, config, func(p relayTestParams) { + r, _ := http.NewRequest("GET", "http://localhost/status", nil) + result, body := st.DoRequest(r, p.relay) + assert.Equal(t, http.StatusOK, result.StatusCode) + status := ldvalue.Parse(body) + + // EnvMain is a manually-configured single-key env: sdkKeys[] is present and contains + // exactly the anchor (full set includes it); manual config carries no key identifier. + sdkKeys := status.GetByKey("environments").GetByKey(st.EnvMain.Name).GetByKey("sdkKeys") + require.Equal(t, 1, sdkKeys.Count(), "sdkKeys must contain the anchor") + assert.Equal(t, sdks.ObscureKey(string(st.EnvMain.Config.SDKKey)), + sdkKeys.GetByIndex(0).GetByKey("value").StringValue()) + + // EnvMain has no mobile key — mobileKeys is present but empty. + mobileKeys := status.GetByKey("environments").GetByKey(st.EnvMain.Name).GetByKey("mobileKeys") + assert.Equal(t, 0, mobileKeys.Count()) + assert.Equal(t, ldvalue.ArrayType, mobileKeys.Type(), "mobileKeys present (not null) even when empty") + + // EnvMobile has both: its mobile key appears in mobileKeys[]. + mobMobileKeys := status.GetByKey("environments").GetByKey(st.EnvMobile.Name).GetByKey("mobileKeys") + require.Equal(t, 1, mobMobileKeys.Count()) + assert.Equal(t, sdks.ObscureKey(string(st.EnvMobile.Config.MobileKey)), + mobMobileKeys.GetByIndex(0).GetByKey("value").StringValue()) + }) + }) }) t.Run("connection interruption - less than DisconnectedStatusTime", func(t *testing.T) { @@ -131,3 +163,116 @@ func TestEndpointsStatus(t *testing.T) { }) }) } + +// findKeyStatusByValue returns the sdkKeys[]/mobileKeys[] entry whose obscured "value" matches, or a +// null value. Array entry order is unspecified, so callers look entries up by value. +func findKeyStatusByValue(arr ldvalue.Value, obscuredValue string) ldvalue.Value { + for i := 0; i < arr.Count(); i++ { + if arr.GetByIndex(i).GetByKey("value").StringValue() == obscuredValue { + return arr.GetByIndex(i) + } + } + return ldvalue.Null() +} + +// TestEndpointsStatusExpiringSDKKey drives a multi-key environment through the real /status handler: +// it reconciles an env to an anchor plus two non-anchor expiring SDK keys and asserts the +// expiringSdkKey selection, the per-key expiry/identifier serialization in sdkKeys[], and that the +// soonest-expiry pick is deterministic on an exact expiry tie. +func TestEndpointsStatusExpiringSDKKey(t *testing.T) { + getStatus := func(t *testing.T, p relayTestParams, set credential.AcceptedSet) ldvalue.Value { + env, err := p.relay.getEnvironment(sdkauth.New(st.EnvMain.Config.SDKKey)) + require.NoError(t, err) + require.NotNil(t, env) + env.ReconcileCredentials(set) + + r, _ := http.NewRequest("GET", "http://localhost/status", nil) + result, body := st.DoRequest(r, p.relay) + require.Equal(t, http.StatusOK, result.StatusCode) + return ldvalue.Parse(body).GetByKey("environments").GetByKey(st.EnvMain.Name) + } + + t.Run("soonest-expiring non-anchor key, with expiry and identifier surfaced", func(t *testing.T) { + var config c.Config + config.Environment = st.MakeEnvConfigs(st.EnvMain) + withStartedRelay(t, config, func(p relayTestParams) { + anchor := st.EnvMain.Config.SDKKey + soon := time.Now().Add(1 * time.Hour) + later := time.Now().Add(2 * time.Hour) + set, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: anchor}). + WithSDKKey(credential.SDKKeyParams{Value: "sdk-soon", Key: util.PtrOrNil("soon-key"), Expiry: util.PtrOrNil(soon)}). + WithSDKKey(credential.SDKKeyParams{Value: "sdk-later", Expiry: util.PtrOrNil(later)}). + Build() + require.NoError(t, err) + + envStatus := getStatus(t, p, set) + + // expiringSdkKey is the obscured soonest-expiring non-anchor key. + st.AssertJSONPathMatch(t, sdks.ObscureKey("sdk-soon"), envStatus, "expiringSdkKey") + + // sdkKeys[] carries the full set (anchor + both non-anchor keys). + sdkKeys := envStatus.GetByKey("sdkKeys") + require.Equal(t, 3, sdkKeys.Count()) + + // The expiring key surfaces its identifier and Unix-millis expiry. + soonEntry := findKeyStatusByValue(sdkKeys, sdks.ObscureKey("sdk-soon")) + require.False(t, soonEntry.IsNull()) + assert.Equal(t, "soon-key", soonEntry.GetByKey("key").StringValue()) + assert.Equal(t, float64(soon.UnixMilli()), soonEntry.GetByKey("expiry").Float64Value()) + + // A key with no identifier omits "key" entirely (omitempty, nil pointer). + laterEntry := findKeyStatusByValue(sdkKeys, sdks.ObscureKey("sdk-later")) + require.False(t, laterEntry.IsNull()) + assert.True(t, laterEntry.GetByKey("key").IsNull(), `"key" must be omitted when the source carried no identifier`) + }) + }) + + t.Run("tie on expiry resolves deterministically to the smaller value", func(t *testing.T) { + var config c.Config + config.Environment = st.MakeEnvConfigs(st.EnvMain) + withStartedRelay(t, config, func(p relayTestParams) { + anchor := st.EnvMain.Config.SDKKey + sameExpiry := time.Now().Add(1 * time.Hour) + set, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: anchor}). + WithSDKKey(credential.SDKKeyParams{Value: "sdk-bbb", Expiry: util.PtrOrNil(sameExpiry)}). + WithSDKKey(credential.SDKKeyParams{Value: "sdk-aaa", Expiry: util.PtrOrNil(sameExpiry)}). + Build() + require.NoError(t, err) + + envStatus := getStatus(t, p, set) + // With equal expiries, the smaller value (sdk-aaa) wins deterministically. + st.AssertJSONPathMatch(t, sdks.ObscureKey("sdk-aaa"), envStatus, "expiringSdkKey") + }) + }) +} + +// TestKeyStatus verifies the helper that converts an accepted key into its status-endpoint JSON form. +func TestKeyStatus(t *testing.T) { + strptr := func(s string) *string { return &s } + + t.Run("permanent key with identifier", func(t *testing.T) { + ks := keyStatus("sdk-abc123", credential.AcceptedKey{Key: strptr("default")}) + assert.Equal(t, "default", ks.Key) + assert.Equal(t, sdks.ObscureKey("sdk-abc123"), ks.Value) + assert.Nil(t, ks.Expiry) + }) + + t.Run("nil identifier yields empty Key (omitted in JSON)", func(t *testing.T) { + ks := keyStatus("sdk-legacy", credential.AcceptedKey{Key: nil}) + assert.Equal(t, "", ks.Key) + }) + + t.Run("expiring key has expiry in Unix milliseconds", func(t *testing.T) { + expiry := time.Date(2099, 6, 1, 12, 0, 0, 0, time.UTC) + ks := keyStatus("sdk-old", credential.AcceptedKey{Key: strptr("old-key"), Expiry: &expiry}) + require.NotNil(t, ks.Expiry) + assert.Equal(t, expiry.UnixMilli(), *ks.Expiry) + }) + + t.Run("mobile key value is obscured", func(t *testing.T) { + ks := keyStatus("mob-secret", credential.AcceptedKey{Key: strptr("mob-1")}) + assert.Equal(t, sdks.ObscureKey("mob-secret"), ks.Value) + }) +} From 211cec19a3d6dc206b437ece055394626a0c12f2 Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Tue, 7 Jul 2026 08:09:17 -0400 Subject: [PATCH 31/66] feat: Hand over the data store to the new client on re-anchor (#736) --- .../relayenv/env_context_reanchor_test.go | 89 ++++---- .../store_handover_realclient_test.go | 204 ++++++++++++++++++ internal/store/relay_feature_store.go | 71 +++++- .../store/store_rebuild_after_close_test.go | 66 ++++++ internal/store/store_refcount_test.go | 148 +++++++++++++ 5 files changed, 525 insertions(+), 53 deletions(-) create mode 100644 internal/relayenv/store_handover_realclient_test.go create mode 100644 internal/store/store_rebuild_after_close_test.go create mode 100644 internal/store/store_refcount_test.go diff --git a/internal/relayenv/env_context_reanchor_test.go b/internal/relayenv/env_context_reanchor_test.go index d967cf96..d3009db3 100644 --- a/internal/relayenv/env_context_reanchor_test.go +++ b/internal/relayenv/env_context_reanchor_test.go @@ -1,19 +1,16 @@ package relayenv -// T0 — Re-anchoring PoC (SDK-2453 / SDK-2530). +// Re-anchoring proof-of-concept tests. // -// These tests validate the upstream SDK-client swap mechanism that T2.c will implement. Each test -// answers one of the seven hypotheses in .agent-docs/concurrent-keys/phase1-design.md §7. They are -// written as durable, executable probes of today's primitives so they survive into T2 as regression -// tests and as the executable spec for the re-anchor implementation. -// -// A written summary of the findings lives in -// .agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md. +// These tests validate the upstream SDK-client swap mechanism that the re-anchor implementation builds +// on. Each test answers one of the seven hypotheses about how re-anchoring should behave. They are +// written as durable, executable probes of today's primitives so they survive as regression tests and +// as the executable spec for the re-anchor implementation. // // Terminology: "re-anchor" = swapping the single upstream SDK client when sdkKey.value changes. // Today there is no dedicated re-anchor method; the closest existing path is ReconcileCredentials with // an expiring (grace-period) key (which rotates the primary SDK key and stands up a new client), so -// several tests drive that path and observe where it falls short of the §7 requirements. +// several tests drive that path and observe where it falls short of the requirements. import ( "errors" @@ -119,12 +116,10 @@ func TestReanchorPoC_H1_SharedStoreAdapterRebuildSemantics(t *testing.T) { featureKind := ldstoreimpl.Features() flagKey := st.Flag1ServerSide.Flag.Key - // The design (§7) assumes "two SDK clients pointed at the same env can feed the same store as a - // side-effect." This sub-test shows that assumption is FALSE for the default in-memory store: each - // client init calls storeAdapter.Build, which constructs a brand-new wrapper around a brand-new - // underlying store and atomically swaps it in. No corruption occurs, but the new client starts from - // an empty store. - t.Run("in-memory factory builds a fresh empty store on each client init", func(t *testing.T) { + // A second storeAdapter.Build hands back the SAME wrapper, with its data still in place, rather + // than constructing a fresh wrapper around a fresh (empty) underlying store. This is what lets a + // re-anchor's new client keep serving the populated store instead of starting empty. + t.Run("in-memory factory reuses the existing store on a second client init (store handover)", func(t *testing.T) { rec := &recordingStreamUpdates{} adapter := store.NewSSERelayDataStoreAdapter(ldcomponents.InMemoryDataStore(), rec) @@ -142,20 +137,19 @@ func TestReanchorPoC_H1_SharedStoreAdapterRebuildSemantics(t *testing.T) { s2, err := adapter.Build(subsystems.BasicClientContext{}) require.NoError(t, err) - // FINDING: Build swaps in a new store instance... - assert.NotSame(t, s1, s2, "each Build creates a new store wrapper") - assert.Same(t, s2, adapter.GetStore(), "the adapter now points at the new store") + // Post-fix: Build hands the existing wrapper to the new client and the adapter still points at it. + assert.Same(t, s1, s2, "store handover: Build returns the existing wrapper") + assert.Same(t, s2, adapter.GetStore(), "the adapter still points at the shared wrapper") - // ...and that store is empty + uninitialized. So the two clients do NOT share data through the - // in-memory store; the new anchor must re-sync from scratch. - assert.False(t, adapter.GetStore().IsInitialized(), "the new in-memory store starts uninitialized") + // The wrapper stays initialized and the data survives — no empty-store window for the new anchor. + assert.True(t, adapter.GetStore().IsInitialized(), "the shared store remains initialized") got2, err := adapter.GetStore().Get(featureKind, flagKey) require.NoError(t, err) - assert.Nil(t, got2.Item, "the new in-memory store starts empty") + assert.NotNil(t, got2.Item, "data persists across handover") }) // With a persistent store, the underlying data lives outside the wrapper, so the swap preserves it. - // This is the configuration in which the §7 "shared store" assumption actually holds. + // This is the configuration in which the "shared store" assumption actually holds. t.Run("shared (persistent) underlying store preserves data across client init", func(t *testing.T) { underlying, err := ldcomponents.InMemoryDataStore().Build(subsystems.BasicClientContext{}) require.NoError(t, err) @@ -273,7 +267,7 @@ func TestReanchorPoC_H2_NewClientInitialSyncRebroadcastsPut(t *testing.T) { // FINDING: the new anchor's initial sync produces a second full "put" to every connected downstream // stream. From a downstream SDK's perspective this is a duplicate put. It is tolerable (SDKs apply - // puts idempotently) but T2.c must expect it; it is not a corruption. + // puts idempotently) but the re-anchor implementation must expect it; it is not a corruption. assert.Equal(t, 2, rec.allDataCount(), "the new anchor's initial sync re-broadcasts a full put") } @@ -367,8 +361,8 @@ func TestReanchorPoC_H3_BigSegmentSyncIsNotReWiredOnReAnchor(t *testing.T) { // FINDING: big-segment sync is wired to the SDK key at construction and is NOT re-wired by today's // swap path -- the synchronizer is neither recreated nor told about the new key (the // BigSegmentSynchronizer interface has no credential-replacement method). After re-anchor it keeps - // polling/streaming on the OLD anchor key. T2.d must add a re-wire path (a ReplaceCredential-style - // method) or recreate the synchronizer on each re-anchor. + // polling/streaming on the OLD anchor key. The big-segment re-wire (follow-up work) must add a + // re-wire path (a ReplaceCredential-style method) or recreate the synchronizer on each re-anchor. count, sdkKey = capturing.snapshot() assert.Equal(t, 1, count, "synchronizer was not recreated on re-anchor") assert.Equal(t, envConfig.SDKKey, sdkKey, "synchronizer still references the old anchor key") @@ -417,7 +411,12 @@ func TestReanchorPoC_H4_HTTPConfigIsKeyIndependentExceptAuthHeader(t *testing.T) // Hypothesis 5: Order of operations / the in-memory store window. // ----------------------------------------------------------------------------------------------- -func TestReanchorPoC_H5_InMemoryStoreIsWipedByReAnchor(t *testing.T) { +// TestReanchorPoC_H5_StoreSurvivesReAnchor asserts that a re-anchor keeps the env's in-memory data +// store instance intact: the same instance, still initialized, with its data preserved. This holds +// because SSERelayDataStoreAdapter.Build hands its existing wrapper over to the new anchor's client +// (with refcounted Close) rather than building a fresh, empty store. It is the end-to-end proof that +// store handover holds through env_context. +func TestReanchorPoC_H5_StoreSurvivesReAnchor(t *testing.T) { featureKind := ldstoreimpl.Features() flagKey := st.Flag1ServerSide.Flag.Key envConfig := st.EnvMain.Config @@ -450,33 +449,27 @@ func TestReanchorPoC_H5_InMemoryStoreIsWipedByReAnchor(t *testing.T) { require.Eventually(t, func() bool { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond, "GetClient should return the new anchor's client once it is registered") - // FINDING: starting the new client replaced the data store with a fresh, empty, uninitialized one. - // This happens regardless of operation order, because building the new client is what rebuilds the - // store. So "start-new -> swap-pointer -> close-old" alone is NOT sufficient with an in-memory store: - // there is a window in which evaluations see an empty store until the new anchor finishes its initial - // sync. T2.c must either (a) keep the old store/anchor authoritative until the new client reports - // Initialized()==true, (b) require a persistent store for graceful re-anchor, or (c) decouple the - // data store lifecycle from the client lifecycle so a new client does not rebuild it. + // Post-fix: the adapter handed the existing wrapper to the new client, so the env's store is the + // same instance, still initialized, and the data is intact. There is no empty-store window for the + // new anchor. newStore := env.GetStore() - assert.NotSame(t, oldStore, newStore, "the data store instance was replaced by the new client") - assert.False(t, newStore.IsInitialized(), "the new store is uninitialized until the new anchor re-syncs") + assert.Same(t, oldStore, newStore, "the data store instance survives re-anchor (store handover)") + assert.True(t, newStore.IsInitialized(), "the store stays initialized across re-anchor") got2, err := newStore.Get(featureKind, flagKey) require.NoError(t, err) - assert.Nil(t, got2.Item, "data is absent in the new store until the new anchor re-syncs") + assert.NotNil(t, got2.Item, "data is preserved across re-anchor") } -// TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow validates the reviewer suggestion that, because -// relay owns the data store implementation (it hands the SDK a single storeAdapter), the re-anchor can -// hand the existing store over to the new client instead of letting it build a fresh one. Modeled here -// by a DataStoreFactory that returns the same underlying store on every Build; the production change -// (T2.c/T2.d) is to make SSERelayDataStoreAdapter reuse its store across the swap. With handover the -// new anchor's client sees the populated, initialized store immediately -- no empty-store window -// (contrast TestReanchorPoC_H5_InMemoryStoreIsWipedByReAnchor). +// TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow exercises store handover at the store layer: +// because relay owns the data store implementation (it hands the SDK a single storeAdapter), the +// re-anchor hands the existing store to the new client instead of letting it build a fresh one. It is +// modeled here by a DataStoreFactory that returns the same underlying store on every Build, so the new +// anchor's client sees the populated, initialized store immediately with no empty-store window. // -// CAVEAT for the implementation (not reproducible with the fake client, so documented here and in the -// findings): streamUpdatesStoreWrapper.Close() closes the underlying store. If the new client wraps the -// SAME underlying store, closing the retiring client must NOT close it -- the store's lifecycle has to -// be owned by the adapter, not by the client being retired. +// The store's lifecycle is owned by the adapter, not the client: streamUpdatesStoreWrapper.Close() +// closes the underlying store, so when the retiring and new clients share one underlying store the +// retiring client's Close() must not tear it down. The fake client cannot exercise the real client's +// Close(), so that half of the contract is covered by TestRealClient_HandoverPreservesUnderlyingStore. func TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow(t *testing.T) { featureKind := ldstoreimpl.Features() flagKey := st.Flag1ServerSide.Flag.Key diff --git a/internal/relayenv/store_handover_realclient_test.go b/internal/relayenv/store_handover_realclient_test.go new file mode 100644 index 00000000..016b9ec7 --- /dev/null +++ b/internal/relayenv/store_handover_realclient_test.go @@ -0,0 +1,204 @@ +package relayenv + +// Spike verifying the real ld.LDClient's Close() behavior against the SSERelayDataStoreAdapter / +// streamUpdatesStoreWrapper pair, which the fake-client PoC could not exercise. This is the single +// remaining piece the fake-client PoC could not validate: +// +// > streamUpdatesStoreWrapper.Close() closes the underlying store. With handover the retiring +// > and new clients share one underlying store, so closing the retiring client must NOT close +// > it — the adapter (not the client) must own the store's lifecycle. (Not reproducible with +// > the fake client; verified here against the real client.) +// +// We answer two questions: +// Q1. Does ld.LDClient.Close() invoke Close() on its data store (the wrapper)? +// Q2. After the wrapper's Close() runs, is the underlying store still usable for reads? +// +// Q1 determines whether store handover is at risk at all. Q2 determines whether the remedy needs to +// gate the wrapper's Close (case A: in-memory Close is destructive) or whether it can stay as-is +// (case B: in-memory Close is a no-op and reads still work). + +import ( + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/internal/store" + st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + + ld "github.com/launchdarkly/go-server-sdk/v7" + "github.com/launchdarkly/go-server-sdk/v7/ldcomponents" + "github.com/launchdarkly/go-server-sdk/v7/subsystems" + "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" + "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoretypes" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// closeObservingStore wraps an in-memory data store factory so we can directly observe when the +// underlying store's Close() is invoked. This is the unambiguous signal that ld.LDClient.Close() +// propagates through streamUpdatesStoreWrapper.Close() to the wrapped store. +type closeObservingStore struct { + inner subsystems.DataStore + closeCount int +} + +func (c *closeObservingStore) Close() error { + c.closeCount++ + return c.inner.Close() +} +func (c *closeObservingStore) Init(d []ldstoretypes.Collection) error { return c.inner.Init(d) } +func (c *closeObservingStore) Get(k ldstoretypes.DataKind, key string) (ldstoretypes.ItemDescriptor, error) { + return c.inner.Get(k, key) +} +func (c *closeObservingStore) GetAll(k ldstoretypes.DataKind) ([]ldstoretypes.KeyedItemDescriptor, error) { + return c.inner.GetAll(k) +} +func (c *closeObservingStore) Upsert(k ldstoretypes.DataKind, key string, item ldstoretypes.ItemDescriptor) (bool, error) { + return c.inner.Upsert(k, key, item) +} +func (c *closeObservingStore) IsInitialized() bool { return c.inner.IsInitialized() } +func (c *closeObservingStore) IsStatusMonitoringEnabled() bool { return c.inner.IsStatusMonitoringEnabled() } + +type closeObservingStoreFactory struct { + observed *closeObservingStore +} + +func (f *closeObservingStoreFactory) Build(ctx subsystems.ClientContext) (subsystems.DataStore, error) { + inner, err := ldcomponents.InMemoryDataStore().Build(ctx) + if err != nil { + return nil, err + } + f.observed = &closeObservingStore{inner: inner} + return f.observed, nil +} + +// realClientUsingAdapter spins up a real ld.LDClient backed by the relay store adapter. Using +// ExternalUpdatesOnly as the data source avoids any network calls (no upstream streaming connection +// is opened), so the test is hermetic. The adapter sees the real client's DataStore.Build() call +// and the real Close() lifecycle on shutdown. +func realClientUsingAdapter(t *testing.T, adapter *store.SSERelayDataStoreAdapter) *ld.LDClient { + t.Helper() + cfg := ld.Config{ + DataStore: adapter, + DataSource: ldcomponents.ExternalUpdatesOnly(), + Events: ldcomponents.NoEvents(), + } + client, err := ld.MakeCustomClient("fake-sdk-key", cfg, 5*time.Second) + require.NoError(t, err) + return client +} + +// TestRealClient_CloseInvokesWrapperClose verifies that closing a real ld.LDClient causes the +// wrapped store's Close() to fire. This is the precondition that makes store handover dangerous — +// if Close did not propagate, there would be no lifecycle hazard to design around. +func TestRealClient_CloseInvokesWrapperClose(t *testing.T) { + factory := &closeObservingStoreFactory{} + rec := &recordingStreamUpdates{} + adapter := store.NewSSERelayDataStoreAdapter(factory, rec) + + client := realClientUsingAdapter(t, adapter) + require.NotNil(t, factory.observed, "the adapter must have built the observed store") + require.Equal(t, 0, factory.observed.closeCount, "no Close yet before client.Close") + + wrapper := adapter.GetStore() + require.NoError(t, wrapper.Init(st.AllData)) + require.True(t, wrapper.IsInitialized()) + + require.NoError(t, client.Close()) + + // The headline finding: closing the real client propagates Close() to the underlying store via + // streamUpdatesStoreWrapper.Close(). If this assertion fails, the lifecycle caveat is not a real + // hazard for this combination and the store-handover fix only needs the Build() reuse, not a + // Close() lifecycle change. + assert.Equal(t, 1, factory.observed.closeCount, + "ld.LDClient.Close should propagate to the underlying data store via the wrapper") +} + +// TestRealClient_ReadsAfterCloseAreStillFunctional asks the second question: after Close runs, is +// the underlying in-memory store still usable for Get? The answer tells us whether the store-handover +// fix needs to actually prevent Close (because reads will fail after it) or whether reads coincidentally +// still work (because the in-memory store's Close is effectively a no-op for read behavior). Even +// if reads happen to work, the fix should still gate Close — relying on undocumented "Close is a +// no-op" behavior is brittle and breaks when persistent stores enter the picture. +func TestRealClient_ReadsAfterCloseAreStillFunctional(t *testing.T) { + factory := &closeObservingStoreFactory{} + rec := &recordingStreamUpdates{} + adapter := store.NewSSERelayDataStoreAdapter(factory, rec) + + client := realClientUsingAdapter(t, adapter) + wrapper := adapter.GetStore() + require.NoError(t, wrapper.Init(st.AllData)) + + featureKind := ldstoreimpl.Features() + flagKey := st.Flag1ServerSide.Flag.Key + + got, err := wrapper.Get(featureKind, flagKey) + require.NoError(t, err) + require.NotNil(t, got.Item, "sanity: data is readable before close") + + require.NoError(t, client.Close()) + + // Read after Close. The outcome here is informational, not a pass/fail design gate: + // - If the read succeeds, the in-memory store's Close is effectively a no-op for queries; the fix + // can still safely gate Close to be defensive (persistent stores may differ). + // - If the read fails, the fix MUST gate Close, since the new anchor would observe a broken store. + gotAfter, errAfter := wrapper.Get(featureKind, flagKey) + t.Logf("Get after client.Close: item=%v err=%v initialized=%v", + gotAfter.Item != nil, errAfter, wrapper.IsInitialized()) +} + +// TestRealClient_HandoverPreservesUnderlyingStore exercises the production store-handover behavior: +// when a second real ld.LDClient is built against the same SSERelayDataStoreAdapter — the re-anchor +// case — the adapter hands the existing wrapper (and underlying store) to the new client rather +// than rebuilding it. Closing the first client must not tear the underlying store down while the +// second client is still holding it; only the final Close releases it. +// +// Sequence: +// 1. Build client1; init data on its wrapper. +// 2. Build client2 — adapter returns the SAME wrapper; no second underlying store is built. +// 3. Close client1 — store stays open because client2 still holds it. +// 4. Close client2 — final release; underlying store closes exactly once. +func TestRealClient_HandoverPreservesUnderlyingStore(t *testing.T) { + factory := &countingStoreFactory{} + rec := &recordingStreamUpdates{} + adapter := store.NewSSERelayDataStoreAdapter(factory, rec) + + client1 := realClientUsingAdapter(t, adapter) + wrapper1 := adapter.GetStore() + require.NoError(t, wrapper1.Init(st.AllData)) + require.Equal(t, 1, factory.buildCount, "one Build for client1") + underlying := factory.lastObserved + + client2 := realClientUsingAdapter(t, adapter) + wrapper2 := adapter.GetStore() + require.Equal(t, 1, factory.buildCount, "client2 reuses the existing wrapper — no second Build") + assert.Same(t, wrapper1, wrapper2, "the adapter hands the same wrapper to both clients") + assert.True(t, wrapper2.IsInitialized(), "the shared store stays initialized across handover") + + require.NoError(t, client1.Close()) + assert.Equal(t, 0, underlying.closeCount, + "client1.Close must NOT tear down the underlying store while client2 still holds it") + + require.NoError(t, client2.Close()) + assert.Equal(t, 1, underlying.closeCount, + "client2.Close is the final release; the underlying store is closed exactly once") +} + +// countingStoreFactory tracks every Build call and exposes the most recently built store so the +// handover test can compare instances across builds. Each build wraps a real in-memory store in a +// closeObservingStore so close events are visible. +type countingStoreFactory struct { + buildCount int + lastObserved *closeObservingStore +} + +func (f *countingStoreFactory) Build(ctx subsystems.ClientContext) (subsystems.DataStore, error) { + inner, err := ldcomponents.InMemoryDataStore().Build(ctx) + if err != nil { + return nil, err + } + observed := &closeObservingStore{inner: inner} + f.buildCount++ + f.lastObserved = observed + return observed, nil +} diff --git a/internal/store/relay_feature_store.go b/internal/store/relay_feature_store.go index 5e3d6a7e..b4805c84 100644 --- a/internal/store/relay_feature_store.go +++ b/internal/store/relay_feature_store.go @@ -67,15 +67,31 @@ func NewSSERelayDataStoreAdapter( } // Build is called by the SDK when the LDClient is being created. +// +// Store handover (concurrent-keys re-anchor): if the adapter already holds a wrapper from +// a prior client construction, that wrapper is returned again instead of building a fresh one. This +// hands the populated, initialized data store over to the new anchor's client during a re-anchor — +// no empty-store window, no re-sync. The wrapper refcounts its holders so the underlying store is +// only torn down by the final Close (see streamUpdatesStoreWrapper.Close). If the parked wrapper has +// already been fully closed (acquire returns false), a fresh one is built rather than resurrecting a +// wrapper whose underlying store is torn down. func (a *SSERelayDataStoreAdapter) Build( context subsystems.ClientContext, ) (subsystems.DataStore, error) { - var sw *streamUpdatesStoreWrapper + a.mu.Lock() + if existing := a.store; existing != nil { + if sw, ok := existing.(*streamUpdatesStoreWrapper); ok && sw.acquire() { + a.mu.Unlock() + return sw, nil + } + } + a.mu.Unlock() + wrappedStore, err := a.wrappedFactory.Build(context) if err != nil { return nil, err // this will cause client initialization to fail immediately } - sw = newStreamUpdatesStoreWrapper( + sw := newStreamUpdatesStoreWrapper( a.updates, wrappedStore, context.GetLogging().Loggers, @@ -93,6 +109,15 @@ type streamUpdatesStoreWrapper struct { store subsystems.DataStore updates streams.EnvStreamUpdates loggers ldlog.Loggers + + // refCount tracks how many SDK clients hold this wrapper. The first holder is implicit + // (count starts at 1 in newStreamUpdatesStoreWrapper). Each handover (Build reuse) calls + // acquire to bump the count; each client's Close decrements. The underlying store is torn + // down only when the count reaches zero, at which point closed is set so a later acquire + // refuses to hand back a wrapper whose underlying store is gone. Guarded by refMu. + refMu sync.Mutex + refCount int + closed bool } func newStreamUpdatesStoreWrapper( @@ -101,14 +126,50 @@ func newStreamUpdatesStoreWrapper( loggers ldlog.Loggers, ) *streamUpdatesStoreWrapper { relayStore := &streamUpdatesStoreWrapper{ - store: baseFeatureStore, - updates: updates, - loggers: loggers, + store: baseFeatureStore, + updates: updates, + loggers: loggers, + refCount: 1, } return relayStore } +// acquire records an additional holder of the wrapper, used by SSERelayDataStoreAdapter.Build when it +// hands this wrapper to a new client during a concurrent-keys re-anchor. It returns false if the +// wrapper has already been fully closed (refCount reached zero and the underlying store was torn +// down); the caller must then build a fresh wrapper rather than resurrect a dead one. +func (sw *streamUpdatesStoreWrapper) acquire() bool { + sw.refMu.Lock() + defer sw.refMu.Unlock() + if sw.closed { + return false + } + sw.refCount++ + return true +} + func (sw *streamUpdatesStoreWrapper) Close() error { + sw.refMu.Lock() + if sw.closed { + // Already fully torn down. A stray extra Close (the SDK's LDClient.Close is not idempotent, so + // this depends on caller discipline) must not decrement below zero and re-satisfy the final + // guard — that would close the underlying store a second time, double-releasing a persistent + // store's connection pool. Close is idempotent past the final release. + sw.refMu.Unlock() + return nil + } + sw.refCount-- + final := sw.refCount <= 0 + if final { + sw.closed = true + } + sw.refMu.Unlock() + if !final { + // Re-anchor handover in progress: another client is still using this underlying store. + // The retiring client's Close must not tear it down — see SSERelayDataStoreAdapter.Build + // for the other half of this contract. + return nil + } return sw.store.Close() } diff --git a/internal/store/store_rebuild_after_close_test.go b/internal/store/store_rebuild_after_close_test.go new file mode 100644 index 00000000..4d91f8d1 --- /dev/null +++ b/internal/store/store_rebuild_after_close_test.go @@ -0,0 +1,66 @@ +package store + +// Regression test for the store-handover refcount contract (concurrent-keys re-anchor). +// +// SSERelayDataStoreAdapter.Build reuses whatever wrapper is parked in a.store so a re-anchor can hand +// the populated store to the new client. The hazard: once the wrapper's refCount reaches zero and its +// underlying store is torn down, a later Build must NOT hand that same, now-closed wrapper back to a +// new client (a use-after-close for a persistent store whose Close releases its connection pool). The +// fix marks a fully-closed wrapper and has acquire refuse it, so Build rebuilds a fresh wrapper. +// +// (The re-anchor flow keeps the anchor client permanent, so refCount doesn't reach zero while the env +// is alive today; this guards the wrapper/adapter contract itself against a future caller.) + +import ( + "testing" + + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + + "github.com/launchdarkly/go-server-sdk/v7/subsystems" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// freshStoreFactory builds a new underlying store on every Build, mirroring a real DataStore factory +// (mockStoreFactory returns a single fixed instance, which can't model a rebuild). +type freshStoreFactory struct { + built []*mockStore +} + +func (f *freshStoreFactory) Build(_ subsystems.ClientContext) (subsystems.DataStore, error) { + s := &mockStore{realStore: sharedtest.NewInMemoryStore()} + f.built = append(f.built, s) + return s, nil +} + +func TestStoreAdapterRebuildsAfterFullClose(t *testing.T) { + factory := &freshStoreFactory{} + updates := &mockEnvStreamsUpdates{} + adapter := NewSSERelayDataStoreAdapter(factory, updates) + ctx := subsystems.BasicClientContext{} + + // First client builds the wrapper: refCount = 1. + first, err := adapter.Build(ctx) + require.NoError(t, err) + sw1 := first.(*streamUpdatesStoreWrapper) + require.Equal(t, 1, sw1.currentRefCount()) + + // The sole client shuts down: refCount 1 -> 0, wrapper marked closed, underlying store torn down. + require.NoError(t, first.Close()) + require.True(t, factory.built[0].closed, "final Close tears down the underlying store") + + // A subsequent Build (e.g. a later re-anchor) must NOT resurrect the fully-closed wrapper — it + // rebuilds a fresh wrapper backed by a fresh, open store. + second, err := adapter.Build(ctx) + require.NoError(t, err) + sw2 := second.(*streamUpdatesStoreWrapper) + + assert.NotSame(t, sw1, sw2, "adapter must rebuild rather than hand back the torn-down wrapper") + assert.Equal(t, 1, sw2.currentRefCount(), "the fresh wrapper starts at refCount 1") + assert.False(t, sw2.store.(*mockStore).closed, "the fresh wrapper's underlying store is open") + assert.Same(t, sw2, adapter.GetStore(), "the adapter now points at the fresh wrapper") + + // acquire on the fully-closed wrapper refuses, so it can never be resurrected. + assert.False(t, sw1.acquire(), "acquire on a fully-closed wrapper must return false") +} diff --git a/internal/store/store_refcount_test.go b/internal/store/store_refcount_test.go new file mode 100644 index 00000000..f74cf586 --- /dev/null +++ b/internal/store/store_refcount_test.go @@ -0,0 +1,148 @@ +package store + +// Refcount contract tests for the store-handover wrapper (concurrent-keys re-anchor). +// +// These cover the two properties the refcount design hinges on but the original suite left +// unexercised (multi-agent review, PR #736): Close idempotency past the final release, and safety of a +// Build-reuse (acquire) racing the retiring client's Close. Run the package with -race. + +import ( + "sync" + "testing" + + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + + "github.com/launchdarkly/go-server-sdk/v7/subsystems" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// countingCloseStore counts how many times Close is invoked on the underlying store, so a double +// teardown is observable. Count is mutex-guarded so the concurrency test can read it safely. +type countingCloseStore struct { + subsystems.DataStore + mu sync.Mutex + count int +} + +func (s *countingCloseStore) Close() error { + s.mu.Lock() + s.count++ + s.mu.Unlock() + return s.DataStore.Close() +} + +func (s *countingCloseStore) closeCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.count +} + +type countingCloseStoreFactory struct { + mu sync.Mutex + built []*countingCloseStore +} + +func (f *countingCloseStoreFactory) Build(_ subsystems.ClientContext) (subsystems.DataStore, error) { + f.mu.Lock() + defer f.mu.Unlock() + s := &countingCloseStore{DataStore: sharedtest.NewInMemoryStore()} + f.built = append(f.built, s) + return s, nil +} + +func (f *countingCloseStoreFactory) allBuilt() []*countingCloseStore { + f.mu.Lock() + defer f.mu.Unlock() + return append([]*countingCloseStore(nil), f.built...) +} + +// currentRefCount reads the wrapper's holder count under refMu, so tests can inspect it while other +// goroutines may be calling acquire/Close. Test-only helper (kept out of the production file so the +// unused linter doesn't flag it). +func (sw *streamUpdatesStoreWrapper) currentRefCount() int { + sw.refMu.Lock() + defer sw.refMu.Unlock() + return sw.refCount +} + +// TestWrapperCloseIsIdempotent: once the final holder has released the wrapper and torn down the +// underlying store, a stray extra Close must be a no-op — not decrement refCount below zero and +// re-close the underlying store (which double-releases a persistent store's connection pool). +func TestWrapperCloseIsIdempotent(t *testing.T) { + factory := &countingCloseStoreFactory{} + adapter := NewSSERelayDataStoreAdapter(factory, &mockEnvStreamsUpdates{}) + + sw, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + + require.NoError(t, sw.Close()) + require.Equal(t, 1, factory.built[0].closeCount(), "final Close tears down the underlying store once") + + require.NoError(t, sw.Close()) + assert.Equal(t, 1, factory.built[0].closeCount(), + "Close is idempotent: a second Close must not re-close the underlying store") +} + +// TestWrapperConcurrentCloseClosesExactlyOnce fires many concurrent Close calls on a single wrapper +// (modelling stray/duplicate client Closes arriving at once) and asserts the underlying store is torn +// down exactly once. This directly exercises the idempotent early-return branch under -race and is +// non-vacuous: against a non-idempotent Close, concurrent duplicates drive refCount past zero and +// re-close the underlying store (closeCount > 1). +func TestWrapperConcurrentCloseClosesExactlyOnce(t *testing.T) { + const iterations = 300 + for i := 0; i < iterations; i++ { + factory := &countingCloseStoreFactory{} + adapter := NewSSERelayDataStoreAdapter(factory, &mockEnvStreamsUpdates{}) + + sw, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + + const closers = 8 + var wg sync.WaitGroup + wg.Add(closers) + for c := 0; c < closers; c++ { + go func() { defer wg.Done(); assert.NoError(t, sw.Close()) }() + } + wg.Wait() + + require.Equal(t, 1, factory.built[0].closeCount(), + "the underlying store must be closed exactly once regardless of duplicate concurrent Closes") + } +} + +// TestWrapperHandoverCloseRace: a re-anchor's second Build (reuse -> acquire, or a fresh rebuild if the +// wrapper is already closed) racing the retiring client's Close must never close a single underlying +// store more than once, and -race must stay clean. +func TestWrapperHandoverCloseRace(t *testing.T) { + const iterations = 500 + for i := 0; i < iterations; i++ { + factory := &countingCloseStoreFactory{} + adapter := NewSSERelayDataStoreAdapter(factory, &mockEnvStreamsUpdates{}) + + first, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(2) + // A re-anchor stands up a second client: Build reuses the live wrapper (acquire) or, if the + // retiring Close already tore it down, rebuilds a fresh one. + go func() { + defer wg.Done() + second, buildErr := adapter.Build(subsystems.BasicClientContext{}) + assert.NoError(t, buildErr) + _ = second + }() + // The retiring client closes concurrently. + go func() { + defer wg.Done() + _ = first.Close() + }() + wg.Wait() + + for _, s := range factory.allBuilt() { + require.LessOrEqual(t, s.closeCount(), 1, "an underlying store was closed more than once") + } + } +} From e94f305d3d333f4ba261012610b1920b1d8769eb Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Wed, 8 Jul 2026 08:14:53 -0400 Subject: [PATCH 32/66] refactor: Extract rebuildEvaluator and registerCredentialMappings (#737) --- internal/relayenv/env_context_impl.go | 63 +++++++++++++++++---------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 7bad18ae..d5246742 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -430,17 +430,13 @@ func (c *envContextImpl) cleanupExpiredCredentials(interval time.Duration) { func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { c.mu.Lock() defer c.mu.Unlock() - c.envStreams.AddCredential(newCredential) - for streamProvider, handlers := range c.handlers { - if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, newCredential)); h != nil { - handlers[newCredential] = h - } - } + + c.registerCredentialMappings(newCredential) // A new SDK key means: // 1. we should start a new SDK client*, but only for the anchor: there is a single upstream - // connection per environment, owned by the anchor key. Non-anchor server keys get envStreams - // + handler bundles above, but no upstream client — matching today's mobile-key behavior. + // connection per environment, owned by the anchor key. Non-anchor server keys get their + // credential mappings registered above, but no upstream client — matching today's mobile-key behavior. // 2. we should tell all event forwarding components that use an SDK key to use the new one, // again only when it is the anchor, since events collapse to the anchor per kind. // A new mobile key does not require starting a new SDK client, but does requiring updating any event forwarding @@ -472,8 +468,21 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { } } } +} - c.connectionMapper.AddConnectionMapping(sdkauth.NewScoped(c.filterKey, newCredential), c) +// registerCredentialMappings wires relay's downstream-facing routing for cred: it registers the +// credential with the env's stream machinery, builds the per-stream-provider HTTP handlers, and adds +// the connection->env mapping, so incoming SDK/client connections that authenticate with cred are +// served by this env. It does NOT start the upstream SDK client or repoint event/metrics forwarding -- +// those are anchor-only concerns owned by the caller. The caller must hold c.mu. +func (c *envContextImpl) registerCredentialMappings(cred credential.SDKCredential) { + c.envStreams.AddCredential(cred) + for streamProvider, handlers := range c.handlers { + if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, cred)); h != nil { + handlers[cred] = h + } + } + c.connectionMapper.AddConnectionMapping(sdkauth.NewScoped(c.filterKey, cred), c) } func (c *envContextImpl) removeCredential(oldCredential credential.SDKCredential) { @@ -526,21 +535,9 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env } c.clients[sdkKey] = client - // The data store instance is created by the SDK when it creates the client. Now that - // we have a data store, we can finish setting up the Evaluator that we'll use for this - // environment. - store := c.storeAdapter.GetStore() - dataProvider := ldstoreimpl.NewDataStoreEvaluatorDataProvider(store, c.loggers) - evalOptions := []ldeval.EvaluatorOption{ - // We're setting EnableSecondaryKey because we may be doing evaluations for client-side SDKs that - // are sending old-style user data with the "secondary" attribute. This option doesn't affect - // evaluations done for newer client-side SDKs that send contexts. - ldeval.EvaluatorOptionEnableSecondaryKey(true), - } - if c.sdkBigSegments != nil { - evalOptions = append(evalOptions, ldeval.EvaluatorOptionBigSegmentProvider(c.sdkBigSegments)) - } - c.evaluator = ldeval.NewEvaluatorWithOptions(dataProvider, evalOptions...) + // The data store instance is created by the SDK when it creates the client. Now that we have a + // data store, we can finish setting up the Evaluator for this environment. + c.rebuildEvaluator() } c.initErr = err c.mu.Unlock() @@ -572,6 +569,24 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env } } +// rebuildEvaluator constructs the environment's Evaluator against the current data store. It is called +// after (re)creating an SDK client, once the store is available. It reads and writes envContextImpl +// fields directly, so the caller must hold c.mu. +// +// EnableSecondaryKey is set because we may evaluate for client-side SDKs sending old-style user data +// with the "secondary" attribute; it has no effect for newer SDKs that send contexts. +func (c *envContextImpl) rebuildEvaluator() { + store := c.storeAdapter.GetStore() + dataProvider := ldstoreimpl.NewDataStoreEvaluatorDataProvider(store, c.loggers) + evalOptions := []ldeval.EvaluatorOption{ + ldeval.EvaluatorOptionEnableSecondaryKey(true), + } + if c.sdkBigSegments != nil { + evalOptions = append(evalOptions, ldeval.EvaluatorOptionBigSegmentProvider(c.sdkBigSegments)) + } + c.evaluator = ldeval.NewEvaluatorWithOptions(dataProvider, evalOptions...) +} + // sdkKeyIsActive reports whether the given SDK key is still a tracked credential -- either the primary // key or one within its deprecation grace period -- according to the rotator. startSDKClient uses this // to avoid installing (and thereby leaking) a client for a key that was revoked while the client was From 3399a38ae7bdb83a6c7b33bfe1a0de7d0d5503ae Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Wed, 8 Jul 2026 08:15:12 -0400 Subject: [PATCH 33/66] feat: Defer the SDK anchor flip to an explicit CommitAnchor (#738) --- internal/credential/rotator.go | 54 ++++++++++++++-- internal/credential/rotator_test.go | 59 ++++++++++++++++-- ...v_context_credential_serialization_test.go | 62 +++++++++++++++++++ internal/relayenv/env_context_impl.go | 40 +++++++++++- 4 files changed, 204 insertions(+), 11 deletions(-) create mode 100644 internal/relayenv/env_context_credential_serialization_test.go diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index db6881da..7d59d9f9 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -220,13 +220,57 @@ func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expiration // The set is assumed well-formed: AcceptedSetBuilder.Build validates that an anchor was designated // (and, because WithAnchor adds the key as it designates it, that the anchor is among the SDK // keys), so Reconcile trusts what it is handed rather than re-validating. -func (r *Rotator) Reconcile(set AcceptedSet, now time.Time) { +// +// Reconcile does NOT flip the SDK anchor pointer when the anchor changes. Instead it reports the +// change in the returned ReconcileResult.AnchorChange so the caller can move the pointer at the right +// moment via CommitAnchor. The accepted-set diff (additions/expirations) is applied as before. +func (r *Rotator) Reconcile(set AcceptedSet, now time.Time) ReconcileResult { r.mu.Lock() defer r.mu.Unlock() - r.reconcileSDKKeys(set, set.anchor, now) + var result ReconcileResult + + previousAnchor := r.anchorKey + newAnchor := set.anchor + if previousAnchor != newAnchor && newAnchor.Defined() { + result.AnchorChange = &AnchorChange{ + PreviousAnchor: previousAnchor, + NewAnchor: newAnchor, + } + } + + r.reconcileSDKKeys(set, now) r.reconcileMobileKeys(set, now) r.reconcileEnvironmentID(set) + + return result +} + +// ReconcileResult reports state changes from Reconcile that the caller must act on outside the normal +// addCredential / removeCredential flow driven by StepTime. +// +// AnchorChange is non-nil when the SDK anchor changed. The rotator does NOT flip its anchor pointer in +// that case -- the caller invokes CommitAnchor to move the pointer once it is ready to do so. +type ReconcileResult struct { + AnchorChange *AnchorChange +} + +// AnchorChange describes an SDK anchor transition produced by Reconcile: the anchor moved from +// PreviousAnchor to NewAnchor. PreviousAnchor is the undefined (empty) key when the environment is +// gaining its first SDK anchor. +type AnchorChange struct { + PreviousAnchor config.SDKKey + NewAnchor config.SDKKey +} + +// CommitAnchor atomically moves the rotator's SDK anchor pointer to the given key. Reconcile +// deliberately does not flip the anchor when it changes; the caller invokes CommitAnchor to move the +// pointer. Aside from Initialize (which establishes the initial anchor), CommitAnchor is the only path +// that moves the anchor pointer. +func (r *Rotator) CommitAnchor(key config.SDKKey) { + r.mu.Lock() + defer r.mu.Unlock() + r.anchorKey = key } // reconcilableKey constrains the generic reconcile helper to a comparable credential (so it can key a @@ -276,7 +320,10 @@ func reconcileAcceptedKeys[K reconcilableKey]( // reconcileAcceptedKeys. The set is trusted as well-formed: BuildAcceptedSet / the builder guarantee // the anchor is present and permanent (WithAnchor forces a nil expiry), so no special handling is // needed here. The caller must hold the write lock. -func (r *Rotator) reconcileSDKKeys(set AcceptedSet, anchor config.SDKKey, now time.Time) { +// +// reconcileSDKKeys does NOT flip r.anchorKey when the anchor changes. Reconcile reports the change via +// ReconcileResult.AnchorChange and the caller invokes CommitAnchor to move the pointer. +func (r *Rotator) reconcileSDKKeys(set AcceptedSet, now time.Time) { desired := make(map[config.SDKKey]AcceptedKey, len(set.sdkKeys)) for key, info := range set.sdkKeys { if info.Expiry != nil && !now.Before(*info.Expiry) { @@ -285,7 +332,6 @@ func (r *Rotator) reconcileSDKKeys(set AcceptedSet, anchor config.SDKKey, now ti desired[key] = info } reconcileAcceptedKeys(desired, r.acceptedSDKKeys, &r.additions, &r.expirations, r.loggers, "SDK key") - r.anchorKey = anchor } // reconcileMobileKeys mirrors reconcileSDKKeys for mobile keys. The set is trusted as well-formed: diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index b89eb56a..918e953f 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -54,7 +54,9 @@ func TestReconcileAnchorOnly(t *testing.T) { anchor := config.SDKKey("anchor") now := time.Now() - r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor})), now) + result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor})), now) + require.NotNil(t, result.AnchorChange, "anchor transition from empty to defined is signaled") + r.CommitAnchor(result.AnchorChange.NewAnchor) additions, expirations := r.StepTime(now) assert.ElementsMatch(t, []SDKCredential{anchor}, additions) @@ -70,8 +72,10 @@ func TestReconcileMultipleSDKKeys(t *testing.T) { other := config.SDKKey("other") now := time.Now() - r.Reconcile( + result := r.Reconcile( mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor}).WithSDKKey(SDKKeyParams{Value: other})), now) + require.NotNil(t, result.AnchorChange) + r.CommitAnchor(result.AnchorChange.NewAnchor) additions, expirations := r.StepTime(now) // Both server keys are accepted; only the anchor is primary. @@ -82,6 +86,49 @@ func TestReconcileMultipleSDKKeys(t *testing.T) { assert.Empty(t, r.DeprecatedCredentials()) } +func TestReconcileDefersAnchorFlipUntilCommit(t *testing.T) { + r := newTestRotator() + first := config.SDKKey("first-anchor") + second := config.SDKKey("second-anchor") + now := time.Now() + + // Establishing the initial anchor is itself a transition from the undefined key. + result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: first})), now) + require.NotNil(t, result.AnchorChange) + assert.Equal(t, config.SDKKey(""), result.AnchorChange.PreviousAnchor) + assert.Equal(t, first, result.AnchorChange.NewAnchor) + r.CommitAnchor(result.AnchorChange.NewAnchor) + require.Equal(t, first, r.AnchorKey()) + + // Re-anchor to a new key while the old one stays valid in a grace period. Reconcile must report + // the change but leave the pointer on the previous anchor until CommitAnchor is called. + result = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: second}). + WithSDKKey(SDKKeyParams{Value: first, Expiry: util.PtrOrNil(now.Add(time.Hour))})), now) + require.NotNil(t, result.AnchorChange) + assert.Equal(t, first, result.AnchorChange.PreviousAnchor) + assert.Equal(t, second, result.AnchorChange.NewAnchor) + assert.Equal(t, first, r.AnchorKey(), "Reconcile must not flip the anchor before CommitAnchor") + + r.CommitAnchor(result.AnchorChange.NewAnchor) + assert.Equal(t, second, r.AnchorKey()) +} + +func TestReconcileWithoutAnchorChangeSignalsNil(t *testing.T) { + r := newTestRotator() + anchor := config.SDKKey("anchor") + now := time.Now() + + result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor})), now) + require.NotNil(t, result.AnchorChange) + r.CommitAnchor(result.AnchorChange.NewAnchor) + + // Reconciling again with the same anchor is not a transition. + result = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor})), now) + assert.Nil(t, result.AnchorChange, "no anchor change when the anchor is unchanged") + assert.Equal(t, anchor, r.AnchorKey()) +} + func TestReconcileMultipleMobileKeys(t *testing.T) { r := newTestRotator() anchor := config.SDKKey("anchor") @@ -273,9 +320,11 @@ func TestReconcileDeExpiryRestoresKey(t *testing.T) { func TestAcceptedKeys(t *testing.T) { t.Run("single anchor plus primary mobile", func(t *testing.T) { r := newTestRotator() - r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). WithAnchor(SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). WithPrimaryMobileKey(MobileKeyParams{Value: "mob-primary", Key: util.PtrOrNil("mob-1")})), time.Unix(0, 0)) + require.NotNil(t, result.AnchorChange) + r.CommitAnchor(result.AnchorChange.NewAnchor) set := r.AcceptedKeys() require.Len(t, set.Server, 1) @@ -298,11 +347,13 @@ func TestAcceptedKeys(t *testing.T) { t.Run("multiple keys include the anchor; expiry populated", func(t *testing.T) { r := newTestRotator() expiry := time.Date(2099, 6, 1, 0, 0, 0, 0, time.UTC) - r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). WithAnchor(SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). WithSDKKey(SDKKeyParams{Value: "sdk-b", Key: util.PtrOrNil("b-service")}). WithSDKKey(SDKKeyParams{Value: "sdk-old", Key: util.PtrOrNil("old-key"), Expiry: util.PtrOrNil(expiry)}). WithPrimaryMobileKey(MobileKeyParams{Value: "mob-primary"})), time.Unix(0, 0)) + require.NotNil(t, result.AnchorChange) + r.CommitAnchor(result.AnchorChange.NewAnchor) set := r.AcceptedKeys() require.Len(t, set.Server, 3) // anchor + sdk-b + sdk-old diff --git a/internal/relayenv/env_context_credential_serialization_test.go b/internal/relayenv/env_context_credential_serialization_test.go new file mode 100644 index 00000000..76931657 --- /dev/null +++ b/internal/relayenv/env_context_credential_serialization_test.go @@ -0,0 +1,62 @@ +package relayenv + +// Regression for the deferred-flip concurrency hazard: the cleanup ticker's triggerCredentialChanges +// must be serialized against reconcileCredentials via reconcileMu. Because Reconcile queues the new +// anchor's addition but defers the pointer flip to CommitAnchor, a ticker that drained that addition in +// the window between them would run addCredential with the anchor still on the old key, skip the new +// anchor's startSDKClient, and leave the env with no upstream client. reconcileMu closes that window. + +import ( + "testing" + "time" + + st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + + "github.com/stretchr/testify/require" +) + +func TestCredentialTickerIsSerializedAgainstReconcile(t *testing.T) { + envConfig := st.EnvMain.Config + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactoryWithChannel(true, clientCh), mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + requireClientReady(t, clientCh) + + envImpl := env.(*envContextImpl) + + // Stand in for an in-flight reconcileCredentials by holding reconcileMu: while it is held, the + // cleanup ticker's triggerCredentialChanges must NOT run (that is exactly the interleaving that would + // steal a queued addition mid-re-anchor). + envImpl.reconcileMu.Lock() + + tickerDone := make(chan struct{}) + go func() { + envImpl.triggerCredentialChanges(time.Unix(3000, 0)) + close(tickerDone) + }() + + select { + case <-tickerDone: + envImpl.reconcileMu.Unlock() + t.Fatal("triggerCredentialChanges ran while reconcileMu was held: the ticker is not serialized against reconcile") + case <-time.After(100 * time.Millisecond): + // Expected: the ticker is blocked on reconcileMu. + } + + // Once the "reconcile" releases the lock, the ticker proceeds. + envImpl.reconcileMu.Unlock() + select { + case <-tickerDone: + case <-time.After(time.Second): + t.Fatal("triggerCredentialChanges did not proceed after reconcileMu was released") + } +} diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index d5246742..f11ec587 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -123,6 +123,12 @@ type envContextImpl struct { connectionMapper ConnectionMapper offline bool closed bool + + // reconcileMu serializes credential reconciliation. Both reconcileCredentials and the cleanup + // ticker's triggerCredentialChanges hold it, so they never interleave: the anchor flip (deferred + // to CommitAnchor) stays atomic with the addition it queues, and the ticker cannot drain a queued + // addition while reconcile has committed the accepted set but not yet moved the anchor pointer. + reconcileMu sync.Mutex } // Implementation of the DataStoreQueries interface that the streams package uses as an abstraction of @@ -621,15 +627,43 @@ func (c *envContextImpl) ReconcileCredentials(newSet credential.AcceptedSet) { // reference time for expiry math; production callers pass time.Now() via ReconcileCredentials. // // The Rotator owns the diff (add → re-anchor → remove) and queues the resulting additions and -// expirations; triggerCredentialChanges then applies them, draining additions before expirations so +// expirations; drainCredentialChanges then applies them, draining additions before expirations so // the accepted set is a superset during the transition. addCredential opens an upstream client only // for the anchor, so non-anchor server keys are accepted and routed without a second connection. +// +// Reconcile no longer flips the SDK anchor pointer itself; it reports an anchor change and the pointer +// is moved here via CommitAnchor. Committing immediately (before the drain) reproduces the previous +// in-place-flip behavior: by the time addCredential runs for the new anchor, AnchorKey() already names +// it, so its upstream client is started exactly as before. +// +// reconcileMu is held across Reconcile → CommitAnchor → drain so the whole sequence is atomic against +// the cleanup ticker's triggerCredentialChanges. Without it, the ticker could drain the queued new-anchor +// addition in the window after Reconcile queued it but before CommitAnchor moved the pointer, so +// addCredential's anchor gate would not fire and the new anchor would never get its upstream client. func (c *envContextImpl) reconcileCredentials(newSet credential.AcceptedSet, now time.Time) { - c.keyRotator.Reconcile(newSet, now) - c.triggerCredentialChanges(now) + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + + result := c.keyRotator.Reconcile(newSet, now) + if result.AnchorChange != nil { + c.keyRotator.CommitAnchor(result.AnchorChange.NewAnchor) + } + c.drainCredentialChanges(now) } +// triggerCredentialChanges is the cleanup ticker's entry point (cleanupExpiredCredentials). It takes +// reconcileMu so it is serialized against reconcileCredentials the same way concurrent reconciles are — +// it never interleaves with an in-flight reconcile's Reconcile/CommitAnchor pair. reconcileCredentials +// must NOT call this (it would re-enter reconcileMu); it calls drainCredentialChanges directly. func (c *envContextImpl) triggerCredentialChanges(now time.Time) { + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + c.drainCredentialChanges(now) +} + +// drainCredentialChanges applies the rotator's queued additions (before expirations, so the accepted +// set is a superset during the transition). The caller must hold reconcileMu. +func (c *envContextImpl) drainCredentialChanges(now time.Time) { additions, expirations := c.keyRotator.StepTime(now) for _, cred := range additions { c.addCredential(cred) From 41bd4324cb6a42e1dcb54155faf1ed006975d2c6 Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Wed, 8 Jul 2026 13:58:20 -0400 Subject: [PATCH 34/66] feat: Synchronous re-anchor of the upstream SDK client (#739) --- .agent-docs/concurrent-keys/phase1-design.md | 31 +- internal/credential/rotator.go | 141 +++- internal/credential/rotator_test.go | 300 ++++++-- internal/relayenv/env_context_impl.go | 381 ++++++--- internal/relayenv/env_context_impl_test.go | 35 +- .../env_context_reanchor_double_test.go | 110 +++ .../env_context_reanchor_rollback_test.go | 71 ++ .../env_context_reanchor_synchronous_test.go | 728 ++++++++++++++++++ .../relayenv/env_context_reanchor_test.go | 123 +-- 9 files changed, 1692 insertions(+), 228 deletions(-) create mode 100644 internal/relayenv/env_context_reanchor_double_test.go create mode 100644 internal/relayenv/env_context_reanchor_rollback_test.go create mode 100644 internal/relayenv/env_context_reanchor_synchronous_test.go diff --git a/.agent-docs/concurrent-keys/phase1-design.md b/.agent-docs/concurrent-keys/phase1-design.md index df01c026..1ac645b4 100644 --- a/.agent-docs/concurrent-keys/phase1-design.md +++ b/.agent-docs/concurrent-keys/phase1-design.md @@ -253,6 +253,25 @@ This is the highest-risk piece of Phase 1. The **T0 PoC** validated the swap mec This order is *necessary* — the PoC found that flipping the pointer too early leaves `GetClient()` nil mid-swap (H6) and breaks the env on init failure (H7) — and *sufficient* only with the store-handling approach below. +### Two re-anchor cases — Case A (new key) vs Case B (already-accepted key) + +The sequence above is **Case A**: the new anchor is a key relay has not previously accepted, so no SDK client exists for it. Relay must build one, hand over the store, wait for `Initialized()`, then flip and re-wire. + +**Case B** is the abbreviated path taken when the new anchor's key **already has a live SDK client** — most commonly a *former* anchor that is still inside its grace period (it was demoted on an earlier rotation, kept alive to serve downstream traffic, and is now being promoted back). In that situation: + +1. **No `Build`.** The existing client is reused as-is — there is no second upstream connection to stand up. +2. **No store handover.** The store the existing client created is already populated and initialized; nothing is handed over because nothing new is constructed. +3. **Atomically flip the rotator's anchor pointer** (`CommitAnchor`) — identical to Case A step 3. +4. **Call `ReplaceCredential`** on the event dispatcher + metrics publisher — identical to Case A step 4. +5. Big-segment sync is re-wired (T2.d), identical to Case A. +6. The retiring anchor's client is closed by the existing `removeCredential` path when its own grace period ends — identical to Case A step 6. + +The two paths **converge after the flip**: steps 3–6 are the same. The only difference is the front of the sequence — Case A builds + initializes + hands over the store; Case B reuses what is already there and does none of that. The caller branches on whether a client already exists for the new anchor (`c.clients[newAnchor] != nil`). + +Because Case B does no client build, there is no init-failure rollback to consider for it — the client it reuses was already initialized and serving. Rollback handling (preserve previous anchor, log a structured error) applies to **Case A only**. + +**Reconcile/additions interaction:** so the synchronous re-anchor owns the new anchor's setup end-to-end, `Rotator.Reconcile` does not flip the anchor itself — it returns a `ReconcileResult.AnchorChange` and the caller invokes `CommitAnchor` at the right moment. In **Case A** the new anchor would otherwise appear in the reconcile's `additions` list and `addCredential` would fire an *async* `startSDKClient` that races the synchronous build — so Reconcile strips the new anchor from `additions` in Case A and the synchronous path installs the peripherals (envStreams, handlers, connection mapping) itself. In **Case B** the new anchor was already accepted, so it was never going to appear in `additions` — no stripping is needed there. + ### The data store: hand the existing store over to the new client An earlier version of this design assumed two SDK clients pointed at the same env would feed the *same* data store as a side-effect. The PoC (H1, H5) showed this is **wrong for the in-memory store**: each SDK client construction calls `storeAdapter.Build()`, which atomically swaps in a *new, empty* store, so the new client would otherwise have to re-sync from scratch (an empty-store window). This affects only the in-memory case; with a persistent store (Redis, DynamoDB) the data lives outside the wrapper and survives the swap. @@ -276,17 +295,17 @@ This is the concrete form of decoupling the store's lifecycle from the client's. ### Failure handling -If the new client fails to initialize, the swap **rolls back**: the rotator's anchor pointer stays on the old key, the previous accepted set is preserved, a structured error is logged, and an alarm is raised. The old anchor's client (still alive in its grace period) continues to serve. This is the §8 atomicity principle applied to re-anchor. +If the new client fails to initialize, the swap **rolls back**: the rotator's anchor pointer stays on the old key (the caller simply does not call `CommitAnchor`), the previous accepted set is preserved, and a structured error is logged. The old anchor's client (still alive in its grace period) continues to serve. This is the §8 atomicity principle applied to re-anchor, and it applies to **Case A only** — Case B reuses an already-initialized client and has nothing to fail. Relay has no dedicated alarm infrastructure today; an `Error`-level structured log (`globalLoggers.Errorf`) is the strongest signal available and is sufficient. ### Consolidated specification for T2.c / T2.d | # | Requirement | Source | Owner | |---|---|---|---| -| 1 | Build + initialize the new anchor client *before* flipping the pointer; flip atomically. | H5, H6 | T2.c | -| 2 | On init failure, roll back to old anchor; preserve previous accepted set; log + alarm. | H7 | T2.c | -| 3 | Hand the existing store over to the new client (adapter reuses its store); ensure the retiring client's `Close()` does not tear down the shared store. | H1, H5 | T2.c | -| 4 | Re-wire big-segment sync on re-anchor (recreate or replace-credential). | H3 | T2.d | -| 5 | Call `ReplaceCredential` on event dispatcher + metrics publisher. | §7 | T2.c (already wired in `addCredential`) | +| 1 | **Case A**: build + initialize the new anchor client *before* flipping the pointer; flip atomically via `CommitAnchor`. **Case B** (new anchor already has a live client): skip the build, reuse it, then flip. | H5, H6 | T2.c | +| 2 | **Case A** init failure: roll back (do not `CommitAnchor`); preserve previous accepted set; log a structured error. Not applicable to Case B. | H7 | T2.c | +| 3 | **Case A**: hand the existing store over to the new client (adapter reuses its store, refcounted so the retiring client's `Close()` does not tear it down). **Case B**: no handover — the store is already populated. | H1, H5 | T2.c | +| 4 | Re-wire big-segment sync on re-anchor (recreate or replace-credential). Same for both cases. | H3 | T2.d | +| 5 | Call `ReplaceCredential` on event dispatcher + metrics publisher (synchronously, in the re-anchor sequence — not via `addCredential`, since Reconcile strips the new anchor from `additions`). Same for both cases. | §7 | T2.c | | 6 | Expect duplicate downstream `put`; retain connections for credentials still in the accepted set. | H2 | T2.c (awareness) | | 7 | No `httpconfig` change. | H4 | n/a | diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 7d59d9f9..558844d6 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -2,6 +2,7 @@ package credential import ( "maps" + "slices" "sync" "time" @@ -193,6 +194,11 @@ func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expiration defer r.mu.Unlock() for key, info := range r.acceptedSDKKeys { + if key == r.anchorKey { + // Never expire the current anchor, even if a stale expiry was left on its entry (a + // rolled-back re-anchor can). Same guard as DeprecatedCredentials. + continue + } if info.Expiry != nil && now.After(*info.Expiry) { r.expireSDKKey(key) } @@ -209,6 +215,42 @@ func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expiration return additions, expirations } +// ReconcileResult signals state changes that the caller must apply synchronously rather than rely +// on the normal addCredential / removeCredential flow driven by StepTime. +// +// AnchorChange is non-nil when the SDK anchor changed during Reconcile. The rotator does NOT flip +// its anchor pointer in that case — the caller must drive the synchronous re-anchor sequence +// (build the new anchor's SDK client if one does not exist, wait for Initialized, then invoke +// CommitAnchor to atomically move the pointer, then call ReplaceCredential on the event dispatcher +// and metrics publisher, then re-wire big-segment sync). +// +// MobilePrimaryRepoint is non-nil when the primary mobile key changed AND the new primary was +// already in the accepted set. In that case it does not appear in StepTime's additions list and +// addCredential's primary-mobile gate will not fire for it, so the caller must invoke +// eventDispatcher.ReplaceCredential synchronously. When nil, either the primary mobile key did not +// change, or it changed to a newly-accepted key — in which case the normal addCredential path +// handles the ReplaceCredential call via the existing gate. +type ReconcileResult struct { + AnchorChange *AnchorChange + MobilePrimaryRepoint *config.MobileKey +} + +// AnchorChange describes an SDK anchor transition produced by Reconcile. +// +// NewAnchorPreviouslyAccepted distinguishes the two re-anchor paths: +// - false (the anchor is a new key): the new anchor was not previously in the accepted set. The +// synchronous re-anchor must register the credential mappings (envStreams, handlers, connection +// mapping), construct and initialize a new SDK client, then invoke CommitAnchor + ReplaceCredential. +// - true (the anchor is a previously-accepted key): the new anchor was already accepted (typically +// a former anchor still in its grace period). Its credential mappings are already registered and a +// client may already exist; the synchronous re-anchor reuses it (or constructs one only if missing +// — see the re-anchor sequence in env_context_impl.go), then invokes CommitAnchor + ReplaceCredential. +type AnchorChange struct { + PreviousAnchor config.SDKKey + NewAnchor config.SDKKey + NewAnchorPreviouslyAccepted bool +} + // Reconcile updates the rotator to match set. The set names its own anchor (the primary SDK key) and // primary mobile key. It diffs the desired accepted set against the current one and queues additions // and expirations (drained by the next StepTime call); keys newly present are accepted, and keys no @@ -221,9 +263,11 @@ func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expiration // (and, because WithAnchor adds the key as it designates it, that the anchor is among the SDK // keys), so Reconcile trusts what it is handed rather than re-validating. // -// Reconcile does NOT flip the SDK anchor pointer when the anchor changes. Instead it reports the -// change in the returned ReconcileResult.AnchorChange so the caller can move the pointer at the right -// moment via CommitAnchor. The accepted-set diff (additions/expirations) is applied as before. +// Reconcile does NOT flip the SDK anchor pointer when the anchor changes — the returned +// ReconcileResult.AnchorChange signals the change so the caller can drive the synchronous re-anchor +// sequence, then call CommitAnchor to atomically move the pointer. When the new anchor is a new key +// (NewAnchorPreviouslyAccepted == false) it is also stripped from additions so that the async +// startSDKClient invocation in addCredential does not race the synchronous client build. func (r *Rotator) Reconcile(set AcceptedSet, now time.Time) ReconcileResult { r.mu.Lock() defer r.mu.Unlock() @@ -233,46 +277,88 @@ func (r *Rotator) Reconcile(set AcceptedSet, now time.Time) ReconcileResult { previousAnchor := r.anchorKey newAnchor := set.anchor if previousAnchor != newAnchor && newAnchor.Defined() { + _, alreadyAccepted := r.acceptedSDKKeys[newAnchor] result.AnchorChange = &AnchorChange{ - PreviousAnchor: previousAnchor, - NewAnchor: newAnchor, + PreviousAnchor: previousAnchor, + NewAnchor: newAnchor, + NewAnchorPreviouslyAccepted: alreadyAccepted, } } r.reconcileSDKKeys(set, now) + + if result.AnchorChange != nil && !result.AnchorChange.NewAnchorPreviouslyAccepted { + // The anchor is a new key: reconcileAcceptedKeys just appended it to r.additions. Strip it — + // the synchronous re-anchor sequence in env_context_impl owns the new anchor's setup + // (credential mappings + client build + flip + ReplaceCredential). If addCredential drained this + // addition normally, its async startSDKClient would race the synchronous build. When the anchor + // is a previously-accepted key it was already in acceptedSDKKeys, so reconcileAcceptedKeys did + // not add it — no strip needed. + r.additions = slices.DeleteFunc(r.additions, func(c SDKCredential) bool { return c == newAnchor }) + } + + previousMobile := r.primaryMobileKey + newMobile := set.primaryMobileKey + var newMobileAlreadyAccepted bool + if newMobile.Defined() { + _, newMobileAlreadyAccepted = r.acceptedMobileKeys[newMobile] + } r.reconcileMobileKeys(set, now) + if previousMobile != newMobile && newMobile.Defined() && newMobileAlreadyAccepted { + // Primary mobile key changed to a key already in the accepted set: addCredential's gate will + // not fire for it (it's not in additions), so the caller must call ReplaceCredential itself. + m := newMobile + result.MobilePrimaryRepoint = &m + } + r.reconcileEnvironmentID(set) return result } -// ReconcileResult reports state changes from Reconcile that the caller must act on outside the normal -// addCredential / removeCredential flow driven by StepTime. +// CommitAnchor atomically moves the rotator's SDK anchor pointer to the given key. The caller +// invokes this once the synchronous re-anchor sequence is ready to flip — i.e. after the new +// anchor's client is built and reports Initialized (when the anchor is a new key) or after confirming +// the existing client will be reused (when the anchor is a previously-accepted key). Until CommitAnchor +// is called, the rotator's anchor stays on the previous key so GetClient() returns the still-serving +// old client and the gate in addCredential does not fire for the pending new anchor. // -// AnchorChange is non-nil when the SDK anchor changed. The rotator does NOT flip its anchor pointer in -// that case -- the caller invokes CommitAnchor to move the pointer once it is ready to do so. -type ReconcileResult struct { - AnchorChange *AnchorChange -} - -// AnchorChange describes an SDK anchor transition produced by Reconcile: the anchor moved from -// PreviousAnchor to NewAnchor. PreviousAnchor is the undefined (empty) key when the environment is -// gaining its first SDK anchor. -type AnchorChange struct { - PreviousAnchor config.SDKKey - NewAnchor config.SDKKey -} - -// CommitAnchor atomically moves the rotator's SDK anchor pointer to the given key. Reconcile -// deliberately does not flip the anchor when it changes; the caller invokes CommitAnchor to move the -// pointer. Aside from Initialize (which establishes the initial anchor), CommitAnchor is the only path -// that moves the anchor pointer. +// Aside from Initialize (which establishes the initial anchor), CommitAnchor is the only path that +// moves the anchor pointer: Reconcile deliberately does not flip it (see reconcileSDKKeys). func (r *Rotator) CommitAnchor(key config.SDKKey) { r.mu.Lock() defer r.mu.Unlock() r.anchorKey = key } +// RevertAnchorChange undoes the accepted-set effects of an AnchorChange whose synchronous re-anchor +// failed and rolled back. Because CommitAnchor was never called, the anchor pointer still names the +// previous anchor; this realigns the accepted set with it so the two don't disagree. +// +// - If the previous anchor is a defined key that was revoked in the same reconcile (it is no longer +// accepted — an immediate revocation rather than a grace demotion), re-admit it as a permanent +// key, since it remains the anchor and keeps serving. If it is still accepted (grace demotion), +// leave it and its expiry untouched. An undefined previous anchor (the env's first SDK key) is +// never admitted. +// - Drop the failed new anchor, but only if it was brand new; a previously-accepted key that was +// promoted and failed stays accepted as the non-anchor key it already was. +func (r *Rotator) RevertAnchorChange(change AnchorChange) { + r.mu.Lock() + defer r.mu.Unlock() + + // Only re-admit a defined previous anchor. When an env gains its first SDK key, the previous anchor + // is the empty (undefined) key — there is nothing to re-admit, and inserting "" would put an + // undefined credential into the accepted set (the rotator otherwise only holds defined keys). + if change.PreviousAnchor.Defined() { + if _, stillAccepted := r.acceptedSDKKeys[change.PreviousAnchor]; !stillAccepted { + r.acceptedSDKKeys[change.PreviousAnchor] = AcceptedKey{} + } + } + if !change.NewAnchorPreviouslyAccepted { + delete(r.acceptedSDKKeys, change.NewAnchor) + } +} + // reconcilableKey constrains the generic reconcile helper to a comparable credential (so it can key a // map) that is also an SDKCredential (so it can be logged and appended to the credential lists). type reconcilableKey interface { @@ -321,8 +407,9 @@ func reconcileAcceptedKeys[K reconcilableKey]( // the anchor is present and permanent (WithAnchor forces a nil expiry), so no special handling is // needed here. The caller must hold the write lock. // -// reconcileSDKKeys does NOT flip r.anchorKey when the anchor changes. Reconcile reports the change via -// ReconcileResult.AnchorChange and the caller invokes CommitAnchor to move the pointer. +// NOTE: reconcileSDKKeys does NOT flip r.anchorKey when the anchor changes. The Reconcile caller +// signals the anchor change via ReconcileResult.AnchorChange and invokes CommitAnchor to move the +// pointer once the synchronous re-anchor sequence is ready (see Reconcile + CommitAnchor). func (r *Rotator) reconcileSDKKeys(set AcceptedSet, now time.Time) { desired := make(map[config.SDKKey]AcceptedKey, len(set.sdkKeys)) for key, info := range set.sdkKeys { diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 918e953f..5c4c87a3 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -59,7 +59,8 @@ func TestReconcileAnchorOnly(t *testing.T) { r.CommitAnchor(result.AnchorChange.NewAnchor) additions, expirations := r.StepTime(now) - assert.ElementsMatch(t, []SDKCredential{anchor}, additions) + // The anchor is stripped from additions — the synchronous re-anchor sequence owns its setup. + assert.Empty(t, additions) assert.Empty(t, expirations) assert.Equal(t, anchor, r.AnchorKey()) assert.ElementsMatch(t, []SDKCredential{anchor}, r.AllCredentials()) @@ -78,57 +79,15 @@ func TestReconcileMultipleSDKKeys(t *testing.T) { r.CommitAnchor(result.AnchorChange.NewAnchor) additions, expirations := r.StepTime(now) - // Both server keys are accepted; only the anchor is primary. - assert.ElementsMatch(t, []SDKCredential{anchor, other}, additions) + // Both server keys are accepted; only the non-anchor server key is in additions (the anchor is + // owned by the synchronous re-anchor sequence in env_context_impl). + assert.ElementsMatch(t, []SDKCredential{other}, additions) assert.Empty(t, expirations) assert.Equal(t, anchor, r.AnchorKey()) assert.ElementsMatch(t, []SDKCredential{anchor, other}, r.AllCredentials()) assert.Empty(t, r.DeprecatedCredentials()) } -func TestReconcileDefersAnchorFlipUntilCommit(t *testing.T) { - r := newTestRotator() - first := config.SDKKey("first-anchor") - second := config.SDKKey("second-anchor") - now := time.Now() - - // Establishing the initial anchor is itself a transition from the undefined key. - result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: first})), now) - require.NotNil(t, result.AnchorChange) - assert.Equal(t, config.SDKKey(""), result.AnchorChange.PreviousAnchor) - assert.Equal(t, first, result.AnchorChange.NewAnchor) - r.CommitAnchor(result.AnchorChange.NewAnchor) - require.Equal(t, first, r.AnchorKey()) - - // Re-anchor to a new key while the old one stays valid in a grace period. Reconcile must report - // the change but leave the pointer on the previous anchor until CommitAnchor is called. - result = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(SDKKeyParams{Value: second}). - WithSDKKey(SDKKeyParams{Value: first, Expiry: util.PtrOrNil(now.Add(time.Hour))})), now) - require.NotNil(t, result.AnchorChange) - assert.Equal(t, first, result.AnchorChange.PreviousAnchor) - assert.Equal(t, second, result.AnchorChange.NewAnchor) - assert.Equal(t, first, r.AnchorKey(), "Reconcile must not flip the anchor before CommitAnchor") - - r.CommitAnchor(result.AnchorChange.NewAnchor) - assert.Equal(t, second, r.AnchorKey()) -} - -func TestReconcileWithoutAnchorChangeSignalsNil(t *testing.T) { - r := newTestRotator() - anchor := config.SDKKey("anchor") - now := time.Now() - - result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor})), now) - require.NotNil(t, result.AnchorChange) - r.CommitAnchor(result.AnchorChange.NewAnchor) - - // Reconciling again with the same anchor is not a transition. - result = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor})), now) - assert.Nil(t, result.AnchorChange, "no anchor change when the anchor is unchanged") - assert.Equal(t, anchor, r.AnchorKey()) -} - func TestReconcileMultipleMobileKeys(t *testing.T) { r := newTestRotator() anchor := config.SDKKey("anchor") @@ -140,8 +99,9 @@ func TestReconcileMultipleMobileKeys(t *testing.T) { mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor}).WithPrimaryMobileKey(MobileKeyParams{Value: mob1}).WithMobileKey(MobileKeyParams{Value: mob2})), now) additions, _ := r.StepTime(now) - // Every mobile key is accepted; the designated one is the primary. - assert.ElementsMatch(t, []SDKCredential{anchor, mob1, mob2}, additions) + // Every mobile key is accepted; the anchor is owned by the synchronous re-anchor (stripped from + // additions). The designated primary mobile key and the other mobile key remain in additions. + assert.ElementsMatch(t, []SDKCredential{mob1, mob2}, additions) assert.Equal(t, mob1, r.MobileKey()) assert.ElementsMatch(t, []SDKCredential{anchor, mob1, mob2}, r.AllCredentials()) } @@ -187,7 +147,8 @@ func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { now) additions, expirations := r.StepTime(now) - assert.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, additions) + // Anchor is stripped from additions (owned by the synchronous re-anchor); other keys flow through. + assert.ElementsMatch(t, []SDKCredential{expiringSDK, mob, expiringMobile}, additions) assert.Empty(t, expirations) // Every key is accepted (still authenticates)... assert.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, r.AllCredentials()) @@ -238,7 +199,8 @@ func TestReconcileExpiringKeysAreEvictedByStepTime(t *testing.T) { WithMobileKey(MobileKeyParams{Value: expiringMobile, Expiry: util.PtrOrNil(expiry)})), now) additions, expirations := r.StepTime(now) - require.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, additions) + // Anchor is stripped from additions (owned by the synchronous re-anchor); other keys flow through. + require.ElementsMatch(t, []SDKCredential{expiringSDK, mob, expiringMobile}, additions) require.Empty(t, expirations) // At the exact expiry, expiry is strict (now must be strictly after), so nothing is dropped yet. @@ -264,15 +226,18 @@ func TestReconcileAlreadyExpiredKeyIsIgnoredOnAdd(t *testing.T) { now := time.Unix(2000, 0) alreadyExpired := now.Add(-time.Hour) - r.Reconcile( + result := r.Reconcile( mustBuild(t, NewAcceptedSetBuilder(). WithAnchor(SDKKeyParams{Value: anchor}). WithSDKKey(SDKKeyParams{Value: staleKey, Expiry: util.PtrOrNil(alreadyExpired)})), now) + require.NotNil(t, result.AnchorChange) + r.CommitAnchor(result.AnchorChange.NewAnchor) additions, expirations := r.StepTime(now) - // Only the anchor is added; the stale key is never accepted. - assert.ElementsMatch(t, []SDKCredential{anchor}, additions) + // The fresh anchor is stripped from additions (the synchronous re-anchor sequence owns its setup), + // and the already-expired stale key is never accepted — so nothing is added. + assert.Empty(t, additions) assert.Empty(t, expirations) assert.ElementsMatch(t, []SDKCredential{anchor}, r.AllCredentials()) } @@ -400,3 +365,232 @@ func TestReconcileClearsStaleKeyIdentifier(t *testing.T) { require.True(t, ok) assert.Nil(t, b.Key, "identifier must be cleared when the new payload carries none") } + +func TestReconcileAnchorChangeToPreviouslyAcceptedKey(t *testing.T) { + // When the anchor moves to a key that was already accepted (a non-anchor server key promoted to + // anchor), the AnchorChange reports NewAnchorPreviouslyAccepted == true, and the key is not queued + // as a new addition (it was already accepted). + r := newTestRotator() + anchor := config.SDKKey("anchor") + other := config.SDKKey("other") + now := time.Now() + + result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: other})), now) + require.NotNil(t, result.AnchorChange) + r.CommitAnchor(result.AnchorChange.NewAnchor) + r.StepTime(now) + + // Move the anchor to `other`, keeping the old anchor accepted. + result = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: other}). + WithSDKKey(SDKKeyParams{Value: anchor})), now) + require.NotNil(t, result.AnchorChange) + assert.Equal(t, anchor, result.AnchorChange.PreviousAnchor) + assert.Equal(t, other, result.AnchorChange.NewAnchor) + assert.True(t, result.AnchorChange.NewAnchorPreviouslyAccepted, "other was already in the accepted set") + + additions, _ := r.StepTime(now) + assert.NotContains(t, additions, SDKCredential(other), "an already-accepted new anchor is not a fresh addition") +} + +func TestReconcileMobilePrimaryRepointToAlreadyAcceptedKey(t *testing.T) { + // Switching the primary mobile key to a key that is already accepted must be signaled via + // MobilePrimaryRepoint, because addCredential's gate won't fire for it (it's not in additions). + r := newTestRotator() + anchor := config.SDKKey("anchor") + mob1 := config.MobileKey("mob1") + mob2 := config.MobileKey("mob2") + now := time.Now() + + result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob1}). + WithMobileKey(MobileKeyParams{Value: mob2})), now) + require.NotNil(t, result.AnchorChange) + r.CommitAnchor(result.AnchorChange.NewAnchor) + r.StepTime(now) + + // Make mob2 (already accepted) the primary. It is not a new addition, so it must be signaled. + result = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob2}). + WithMobileKey(MobileKeyParams{Value: mob1})), now) + assert.Nil(t, result.AnchorChange, "the anchor did not change") + require.NotNil(t, result.MobilePrimaryRepoint, "an already-accepted new primary mobile key must be signaled") + assert.Equal(t, mob2, *result.MobilePrimaryRepoint) +} + +func TestReconcileMobilePrimaryToNewKeyDoesNotSignalRepoint(t *testing.T) { + // When the new primary mobile key was NOT already accepted, it arrives via the additions list, so + // addCredential's gate handles the ReplaceCredential and MobilePrimaryRepoint stays nil. + r := newTestRotator() + anchor := config.SDKKey("anchor") + mob1 := config.MobileKey("mob1") + mob2 := config.MobileKey("mob2") + now := time.Now() + + result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob1})), now) + require.NotNil(t, result.AnchorChange) + r.CommitAnchor(result.AnchorChange.NewAnchor) + r.StepTime(now) + + result = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob2})), now) + assert.Nil(t, result.MobilePrimaryRepoint, "a brand-new primary mobile key is handled via additions, not the repoint signal") + + additions, _ := r.StepTime(now) + assert.Contains(t, additions, SDKCredential(mob2), "a brand-new primary mobile key is queued as an addition") +} + +func TestRevertAnchorChangeReadmitsRevokedPreviousAnchor(t *testing.T) { + // When the previous anchor was immediately revoked in the same reconcile (dropped from the accepted + // set) and the re-anchor rolled back, RevertAnchorChange re-admits it and drops the failed new key. + r := newTestRotator() + keyA := config.SDKKey("keyA") + keyB := config.SDKKey("keyB") + now := time.Now() + + res := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: keyA})), now) + require.NotNil(t, res.AnchorChange) + r.CommitAnchor(res.AnchorChange.NewAnchor) + r.StepTime(now) + + // Move the anchor to keyB, omitting keyA entirely (immediate revocation). Reconcile drops keyA. + res = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: keyB})), now) + require.NotNil(t, res.AnchorChange) + require.False(t, res.AnchorChange.NewAnchorPreviouslyAccepted) + + // Simulate the rollback: the caller did NOT CommitAnchor, so the pointer still names keyA. + r.RevertAnchorChange(*res.AnchorChange) + + assert.Equal(t, keyA, r.AnchorKey()) + creds := r.AllCredentials() + assert.Contains(t, creds, SDKCredential(keyA), "previous anchor re-admitted") + assert.NotContains(t, creds, SDKCredential(keyB), "failed new anchor dropped") +} + +func TestRevertAnchorChangeLeavesGraceDemotedPreviousAnchorUntouched(t *testing.T) { + // When the previous anchor was demoted with a grace expiry (still accepted) rather than revoked, + // RevertAnchorChange leaves it — and its expiry — untouched (it does not re-admit it as permanent). + // That is safe because StepTime refuses to expire the current anchor even when its entry carries a + // stale expiry (see TestStepTimeDoesNotExpireCurrentAnchor); RevertAnchorChange itself does not need + // to clear the expiry. + r := newTestRotator() + keyA := config.SDKKey("keyA") + keyB := config.SDKKey("keyB") + now := time.Unix(1000, 0) + expiry := now.Add(time.Hour) + + res := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: keyA})), now) + require.NotNil(t, res.AnchorChange) + r.CommitAnchor(res.AnchorChange.NewAnchor) + r.StepTime(now) + + // Move the anchor to keyB while keeping keyA accepted with a grace expiry. + res = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: keyB}). + WithSDKKey(SDKKeyParams{Value: keyA, Expiry: util.PtrOrNil(expiry)})), now) + require.NotNil(t, res.AnchorChange) + + r.RevertAnchorChange(*res.AnchorChange) + + set := r.AcceptedKeys() + kaInfo, ok := set.Server[keyA] + require.True(t, ok, "grace-demoted previous anchor stays accepted") + require.NotNil(t, kaInfo.Expiry, "RevertAnchorChange must not wipe the grace expiry / make it permanent") + assert.Equal(t, expiry, *kaInfo.Expiry) + _, keyBAccepted := set.Server[keyB] + assert.False(t, keyBAccepted, "failed new anchor dropped") +} + +// TestStepTimeDoesNotExpireCurrentAnchor is the rotator-level guard for the re-anchor-rollback outage: +// even if the current anchor's accepted entry carries an expiry (as it does after a grace-demotion +// re-anchor rolls back before CommitAnchor), StepTime must not expire it. Non-vacuous: without the +// anchor guard in StepTime, this returns keyA in expirations and drops it from the accepted set. +func TestStepTimeDoesNotExpireCurrentAnchor(t *testing.T) { + r := newTestRotator() + keyA := config.SDKKey("keyA") + keyB := config.SDKKey("keyB") + now := time.Unix(1000, 0) + expiry := now.Add(time.Hour) + + res := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: keyA})), now) + require.NotNil(t, res.AnchorChange) + r.CommitAnchor(res.AnchorChange.NewAnchor) + r.StepTime(now) + + // Re-anchor A->B with A grace-demoted; then roll back (never CommitAnchor(keyB), and RevertAnchorChange + // leaves A's expiry). A is still the anchor but now carries a grace expiry. + res = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: keyB}). + WithSDKKey(SDKKeyParams{Value: keyA, Expiry: util.PtrOrNil(expiry)})), now) + require.NotNil(t, res.AnchorChange) + r.RevertAnchorChange(*res.AnchorChange) + require.Equal(t, keyA, r.AnchorKey(), "anchor stayed on keyA after rollback") + + // The cleanup ticker fires past the grace deadline. The anchor must survive. + _, expirations := r.StepTime(expiry.Add(time.Minute)) + + assert.NotContains(t, expirations, SDKCredential(keyA), "StepTime must not expire the current anchor") + assert.Contains(t, r.AllCredentials(), SDKCredential(keyA), "the anchor stays accepted") + assert.Equal(t, keyA, r.AnchorKey()) +} + +// TestStepTimeExpiresDemotedFormerAnchorAfterSuccessfulReanchor pins the narrowness of the anchor guard: +// it protects only the CURRENT anchor. After a SUCCESSFUL re-anchor A->B (CommitAnchor moved the pointer +// to B), the grace-demoted former anchor A is no longer r.anchorKey, so StepTime expires it normally once +// its grace window passes -- the old client must not be pinned alive forever. +func TestStepTimeExpiresDemotedFormerAnchorAfterSuccessfulReanchor(t *testing.T) { + r := newTestRotator() + keyA := config.SDKKey("keyA") + keyB := config.SDKKey("keyB") + now := time.Unix(1000, 0) + expiry := now.Add(time.Hour) + + res := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: keyA})), now) + require.NotNil(t, res.AnchorChange) + r.CommitAnchor(res.AnchorChange.NewAnchor) + r.StepTime(now) + + // Successful re-anchor A->B: the pointer moves to B; A is grace-demoted. + res = r.Reconcile(mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: keyB}). + WithSDKKey(SDKKeyParams{Value: keyA, Expiry: util.PtrOrNil(expiry)})), now) + require.NotNil(t, res.AnchorChange) + r.CommitAnchor(res.AnchorChange.NewAnchor) + require.Equal(t, keyB, r.AnchorKey()) + + // Past A's grace window: A (a non-anchor demoted key now) expires; B (the anchor) survives. + _, expirations := r.StepTime(expiry.Add(time.Minute)) + assert.Contains(t, expirations, SDKCredential(keyA), "the demoted former anchor expires normally") + assert.NotContains(t, r.AllCredentials(), SDKCredential(keyA), "and is dropped from the accepted set") + assert.Contains(t, r.AllCredentials(), SDKCredential(keyB), "the new anchor survives") + assert.Equal(t, keyB, r.AnchorKey()) +} + +func TestRevertAnchorChangeDoesNotAdmitUndefinedPreviousAnchor(t *testing.T) { + // When an env gains its first SDK key, the AnchorChange's previous anchor is the empty (undefined) + // key. A rollback must not insert that empty key into the accepted set. + r := newTestRotator() + keyB := config.SDKKey("keyB") + now := time.Now() + + res := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: keyB})), now) + require.NotNil(t, res.AnchorChange) + require.False(t, res.AnchorChange.PreviousAnchor.Defined(), "the first SDK key has an undefined previous anchor") + + // Simulate a failed build / rollback. + r.RevertAnchorChange(*res.AnchorChange) + + for _, cred := range r.AllCredentials() { + if sdkKey, ok := cred.(config.SDKKey); ok { + assert.True(t, sdkKey.Defined(), "revert must not insert an undefined SDK key into the accepted set") + } + } + assert.NotContains(t, r.AllCredentials(), SDKCredential(keyB), "failed new anchor dropped") +} diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index f11ec587..69f1fcff 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -124,11 +124,18 @@ type envContextImpl struct { offline bool closed bool - // reconcileMu serializes credential reconciliation. Both reconcileCredentials and the cleanup - // ticker's triggerCredentialChanges hold it, so they never interleave: the anchor flip (deferred - // to CommitAnchor) stays atomic with the addition it queues, and the ticker cannot drain a queued - // addition while reconcile has committed the accepted set but not yet moved the anchor pointer. + // reconcileMu serializes reconcileCredentials calls — only one runs at a time, including the + // synchronous re-anchor sequence inside it. Held separately from mu so that GetClient / GetStore / + // GetEvaluator / addCredential continue to run during the (potentially seconds-long) SDK client + // construction when re-anchoring to a new key. reconcileMu sync.Mutex + + // anchorClientGen counts how many times the upstream anchor client has been (re)established. A + // re-anchor commit bumps it. startSDKClient builds its client without c.mu, so a slow build can + // finish after a later re-anchor already installed a fresh anchor client; it captures this value at + // launch and, on completion, discards its (now stale) build if the generation has advanced rather + // than clobbering the current anchor client. Guarded by c.mu. + anchorClientGen uint64 } // Implementation of the DataStoreQueries interface that the streams package uses as an abstraction of @@ -406,7 +413,9 @@ func NewEnvContext( } // Connecting may take time, so do this in parallel - go envContext.startSDKClient(envConfig.SDKKey, readyCh, allConfig.Main.IgnoreConnectionErrors) + // launchGen is 0 here: no re-anchor can have committed yet (the env isn't wired into reconcile until + // after construction returns), so this initial build is never superseded and its result is recorded. + go envContext.startSDKClient(envConfig.SDKKey, readyCh, allConfig.Main.IgnoreConnectionErrors, 0) cleanupInterval := params.ExpiredCredentialCleanupInterval if cleanupInterval == 0 { // 0 means it wasn't specified; the config system disallows 0 as a valid value. @@ -455,7 +464,7 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { case config.SDKKey: if key == c.keyRotator.AnchorKey() { if !c.offline { - go c.startSDKClient(key, nil, false) + go c.startSDKClient(key, nil, false, c.anchorClientGen) } if c.metricsEventPub != nil { // metrics event publisher always uses SDK key c.metricsEventPub.ReplaceCredential(key) @@ -476,21 +485,6 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { } } -// registerCredentialMappings wires relay's downstream-facing routing for cred: it registers the -// credential with the env's stream machinery, builds the per-stream-provider HTTP handlers, and adds -// the connection->env mapping, so incoming SDK/client connections that authenticate with cred are -// served by this env. It does NOT start the upstream SDK client or repoint event/metrics forwarding -- -// those are anchor-only concerns owned by the caller. The caller must hold c.mu. -func (c *envContextImpl) registerCredentialMappings(cred credential.SDKCredential) { - c.envStreams.AddCredential(cred) - for streamProvider, handlers := range c.handlers { - if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, cred)); h != nil { - handlers[cred] = h - } - } - c.connectionMapper.AddConnectionMapping(sdkauth.NewScoped(c.filterKey, cred), c) -} - func (c *envContextImpl) removeCredential(oldCredential credential.SDKCredential) { c.mu.Lock() defer c.mu.Unlock() @@ -512,43 +506,52 @@ func (c *envContextImpl) removeCredential(oldCredential credential.SDKCredential } } -func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- EnvContext, suppressErrors bool) { +func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- EnvContext, suppressErrors bool, launchGen uint64) { client, err := c.sdkClientFactory(sdkKey, c.sdkConfig, c.sdkInitTimeout) c.mu.Lock() name := c.identifiers.GetDisplayName() + // The build happens before we take c.mu. By now the env may be closed, the key may have been + // revoked, or a re-anchor may have committed a fresh anchor client since this build was launched + // (anchorClientGen advanced). In any of those cases this build is stale: close it rather than install + // it, so it cannot clobber the current anchor client. This must not rely on client==nil: a failed SDK + // build returns a non-nil, uninitialized client together with the error, so a stale failed build + // would otherwise replace a healthy anchor client with a dead one. Only defined keys are + // revocation-checked: an undefined SDK key is never tracked, and dropping its client would break envs + // that legitimately run without an SDK key (offline / not-yet-configured / tests). + superseded := c.anchorClientGen != launchGen droppedInactive := false - if client != nil && (c.closed || (sdkKey.Defined() && !c.sdkKeyIsActive(sdkKey))) { - // startSDKClient builds the client before taking c.mu, so by the time we hold the lock the key - // may already have been revoked (rotated away) or the environment may have been closed. In - // either case the freshly-built client must be closed here rather than installed, otherwise its - // upstream connection and goroutines leak until env.Close() (and, once closed, nothing ever - // closes it). removeCredential cannot close it because the client was never in c.clients. - // - // The revocation check applies only to a defined key: an undefined (empty) SDK key is never a - // tracked credential -- the rotator filters undefined credentials out of its accepted set -- so - // it can never be "revoked", and dropping its client would break environments that legitimately - // run without an SDK key (e.g. offline or not-yet-configured envs, and test fixtures). + if client != nil && (c.closed || superseded || (sdkKey.Defined() && !c.sdkKeyIsActive(sdkKey))) { _ = client.Close() client = nil droppedInactive = true } if client != nil { - // If a client already exists for this SDK key (e.g. the key was re-anchored back into the - // primary slot while a previous client for it was still alive in its grace period), close - // the stale one before replacing it so its upstream connection and goroutines are not leaked. + // If a client already exists for this key (e.g. it was re-anchored back into the anchor slot + // while a prior client for it was still in its grace period), close the stale one before + // replacing it so its upstream connection and goroutines are not leaked. if existing := c.clients[sdkKey]; existing != nil && existing != client { _ = existing.Close() } c.clients[sdkKey] = client - - // The data store instance is created by the SDK when it creates the client. Now that we have a - // data store, we can finish setting up the Evaluator for this environment. - c.rebuildEvaluator() + c.rebuildEvaluator() // the SDK created the data store during Build; wire the evaluator to it now + } + // Record this build's result as the env's init status only when it is the current anchor's build: not + // superseded by a newer anchor client, and its key is still the anchor. A genuine failure of the + // current anchor is thus recorded (the middleware 401s a broken env); a stale build's late failure is + // not, so it cannot 401 a healthy re-anchored env. + if !superseded && sdkKey == c.keyRotator.AnchorKey() { + c.initErr = err } - c.initErr = err c.mu.Unlock() switch { + case droppedInactive: + // The build finished but was superseded by a re-anchor, or its key was revoked, or the env + // closed, so it was discarded above rather than installed (even if it also errored -- a + // discarded build's error is moot). The environment is still consistent: no stale client left + // behind. + c.globalLoggers.Infof("SDK key %s build was superseded, revoked, or the environment was closed "+ + "before it finished initializing; the client was discarded", sdkKey.Masked()) case err != nil: if suppressErrors { c.globalLoggers.Warnf("Ignoring error initializing LaunchDarkly client for %q: %+v", @@ -561,12 +564,6 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env } return } - case droppedInactive: - // The client initialized successfully but the key was revoked (or the environment was closed) - // before it could be installed, so it was discarded above. The environment is still considered - // ready: it is in a consistent state with no client for this no-longer-tracked key. - c.globalLoggers.Infof("SDK key %s was revoked or the environment was closed before its client "+ - "finished initializing; the client was discarded", sdkKey.Masked()) default: c.globalLoggers.Infof("Initialized LaunchDarkly client for %q (SDK key %s)", name, sdkKey.Masked()) } @@ -575,28 +572,9 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env } } -// rebuildEvaluator constructs the environment's Evaluator against the current data store. It is called -// after (re)creating an SDK client, once the store is available. It reads and writes envContextImpl -// fields directly, so the caller must hold c.mu. -// -// EnableSecondaryKey is set because we may evaluate for client-side SDKs sending old-style user data -// with the "secondary" attribute; it has no effect for newer SDKs that send contexts. -func (c *envContextImpl) rebuildEvaluator() { - store := c.storeAdapter.GetStore() - dataProvider := ldstoreimpl.NewDataStoreEvaluatorDataProvider(store, c.loggers) - evalOptions := []ldeval.EvaluatorOption{ - ldeval.EvaluatorOptionEnableSecondaryKey(true), - } - if c.sdkBigSegments != nil { - evalOptions = append(evalOptions, ldeval.EvaluatorOptionBigSegmentProvider(c.sdkBigSegments)) - } - c.evaluator = ldeval.NewEvaluatorWithOptions(dataProvider, evalOptions...) -} - -// sdkKeyIsActive reports whether the given SDK key is still a tracked credential -- either the primary -// key or one within its deprecation grace period -- according to the rotator. startSDKClient uses this -// to avoid installing (and thereby leaking) a client for a key that was revoked while the client was -// being constructed. +// sdkKeyIsActive reports whether the given SDK key is still a tracked credential -- the anchor or a key +// within its deprecation grace period -- according to the rotator. startSDKClient uses this to avoid +// installing (and thereby leaking) a client for a key that was revoked while it was being built. func (c *envContextImpl) sdkKeyIsActive(sdkKey config.SDKKey) bool { return slices.Contains(c.keyRotator.AllCredentials(), credential.SDKCredential(sdkKey)) } @@ -623,47 +601,260 @@ func (c *envContextImpl) ReconcileCredentials(newSet credential.AcceptedSet) { c.reconcileCredentials(newSet, time.Now()) } -// reconcileCredentials is the time-injectable implementation of ReconcileCredentials. now is the -// reference time for expiry math; production callers pass time.Now() via ReconcileCredentials. -// -// The Rotator owns the diff (add → re-anchor → remove) and queues the resulting additions and -// expirations; drainCredentialChanges then applies them, draining additions before expirations so -// the accepted set is a superset during the transition. addCredential opens an upstream client only -// for the anchor, so non-anchor server keys are accepted and routed without a second connection. +// reconcileCredentials is the time-injectable implementation of ReconcileCredentials (now is the +// reference time for expiry math). // -// Reconcile no longer flips the SDK anchor pointer itself; it reports an anchor change and the pointer -// is moved here via CommitAnchor. Committing immediately (before the drain) reproduces the previous -// in-place-flip behavior: by the time addCredential runs for the new anchor, AnchorKey() already names -// it, so its upstream client is started exactly as before. +// Order: add -> re-anchor -> remove. Adding first registers the new keys' mappings; the re-anchor then +// swaps the upstream client while the old anchor is still serving; removing last tears down the old +// anchor (and any revoked keys) only once the new one is up. addCredential opens an upstream client +// only for the anchor -- non-anchor server keys are routed without a second connection. // -// reconcileMu is held across Reconcile → CommitAnchor → drain so the whole sequence is atomic against -// the cleanup ticker's triggerCredentialChanges. Without it, the ticker could drain the queued new-anchor -// addition in the window after Reconcile queued it but before CommitAnchor moved the pointer, so -// addCredential's anchor gate would not fire and the new anchor would never get its upstream client. +// reconcileMu serializes this whole method against concurrent reconciles and the cleanup ticker (see +// triggerCredentialChanges). See reanchor for the SDK-anchor swap; MobilePrimaryRepoint is handled +// inline below (a primary-mobile change to an already-accepted key isn't in additions, so addCredential +// won't repoint event forwarding for it). func (c *envContextImpl) reconcileCredentials(newSet credential.AcceptedSet, now time.Time) { c.reconcileMu.Lock() defer c.reconcileMu.Unlock() result := c.keyRotator.Reconcile(newSet, now) + additions, expirations := c.keyRotator.StepTime(now) + + for _, cred := range additions { + c.addCredential(cred) + } + if result.AnchorChange != nil { - c.keyRotator.CommitAnchor(result.AnchorChange.NewAnchor) + if committed := c.reanchor(result.AnchorChange); !committed { + // Rolled back: the new anchor's client never came up. Undo just this anchor change (other + // changes in the payload stand), mirroring RevertAnchorChange. A brand-new anchor had its + // mappings registered this cycle, so tear them down here; a previously-accepted anchor keeps + // its mappings and reverts to the non-anchor key it already was. + if !result.AnchorChange.NewAnchorPreviouslyAccepted { + c.removeCredential(result.AnchorChange.NewAnchor) + } + c.keyRotator.RevertAnchorChange(*result.AnchorChange) + // Keep the previous anchor's client serving by not expiring it here — even if this payload + // revoked it outright. (A grace-demoted previous anchor isn't in expirations anyway, so this + // only matters for an immediate revocation.) + previousAnchor := result.AnchorChange.PreviousAnchor + expirations = slices.DeleteFunc(expirations, func(cred credential.SDKCredential) bool { + return cred == previousAnchor + }) + } + } + + if result.MobilePrimaryRepoint != nil { + c.mu.RLock() + dispatcher := c.eventDispatcher + c.mu.RUnlock() + if dispatcher != nil { + dispatcher.ReplaceCredential(*result.MobilePrimaryRepoint) + } + } + + for _, cred := range expirations { + c.removeCredential(cred) } - c.drainCredentialChanges(now) } -// triggerCredentialChanges is the cleanup ticker's entry point (cleanupExpiredCredentials). It takes -// reconcileMu so it is serialized against reconcileCredentials the same way concurrent reconciles are — -// it never interleaves with an in-flight reconcile's Reconcile/CommitAnchor pair. reconcileCredentials -// must NOT call this (it would re-enter reconcileMu); it calls drainCredentialChanges directly. +// reanchor drives the synchronous re-anchor sequence for an SDK anchor change signaled by Reconcile's +// ReconcileResult.AnchorChange. Invoked by reconcileCredentials after additions have been processed +// and before expirations, so the previous anchor's client is still alive while the new client is built +// (or reused). +// +// reanchor holds c.mu for the whole sequence and releases it only around the SDK client build (which +// must not hold the lock — see buildNewAnchorClient). Holding one continuous lock otherwise keeps +// Close() (which also takes c.mu) from tearing down clients or the dispatcher mid-commit, and lets +// commitReanchor assume the lock is held rather than re-acquiring it. +// +// - When there is no existing client for the new anchor and the env is online: register its credential +// mappings if the key is brand new (Reconcile stripped it from additions), build a new SDK client, +// and on Initialized commit the anchor. On init failure, roll back: do not commit, leave the previous +// anchor authoritative (its client keeps serving), and log a structured error. +// - When a client already exists (e.g. a former anchor still in its grace period), or the env is +// offline: no build, just commit. +// +// Returns true if the anchor was committed, false if it rolled back (init failure or the env closed +// mid-build), so reconcileCredentials can back out the anchor change. The old anchor's client is not +// closed here; its grace-period expiration drives removeCredential. +func (c *envContextImpl) reanchor(change *credential.AnchorChange) bool { + newAnchor := change.NewAnchor + previousAnchor := change.PreviousAnchor + + c.mu.Lock() + defer c.mu.Unlock() + + // Two independent questions: + // - NewAnchorPreviouslyAccepted: are this key's credential mappings already registered? A brand-new + // anchor was stripped from additions, so register them now; an already-accepted key already has them. + // - the client check below: does a client already exist? If so reuse it, else build one. + // They differ for a previously-accepted non-anchor key promoted to anchor: mappings exist, client + // does not. (A live client always implies the key was already accepted, so registration never double-fires.) + if !change.NewAnchorPreviouslyAccepted { + c.registerCredentialMappings(newAnchor) + } + + // A live client for the new anchor is reused as-is; an offline env has no upstream client to build. + // Either way there is nothing to build, so fall through to commit. + why := "reused existing client" + if c.clients[newAnchor] == nil { + if c.offline { + why = "offline — no client build" + } else { + // Build the new client without the lock: sdkClientFactory can block for up to sdkInitTimeout, + // and holding c.mu that long would stall every GetClient/GetStore caller (see reconcileMu). + // reanchor's deferred Unlock releases the lock we re-acquire here on return. + c.mu.Unlock() + client := c.buildNewAnchorClient(newAnchor, previousAnchor) + c.mu.Lock() + + if client == nil { + // Init failed; buildNewAnchorClient already logged and closed the half-built client. Do not + // commit — leave the previous anchor authoritative. + return false + } + if c.closed { + // The env was torn down while the lock was released for the build (Close() does not hold + // reconcileMu, so it can run concurrently); its client-teardown loop has already finished and + // would never close this one, so discard the freshly-built client rather than install it into + // a closed env (mirrors the guard in startSDKClient). + _ = client.Close() + return false + } + if existing := c.clients[newAnchor]; existing != nil && existing != client { + // Stale-client guard: the lock was released for the build, so re-check and close any client + // installed concurrently for newAnchor. + _ = existing.Close() + } + c.clients[newAnchor] = client + // With store handover, GetStore() returns the SAME wrapper the old client used, so the rebuilt + // evaluator serves the already-populated data immediately (no empty-store window). + c.rebuildEvaluator() + why = "built new client" + } + } + + return c.commitReanchor(newAnchor, previousAnchor, why) +} + +// buildNewAnchorClient constructs the SDK client for a re-anchor to newAnchor. It must run without c.mu +// held: sdkClientFactory can block for up to sdkInitTimeout, and holding the lock that long would stall +// every GetClient/GetStore caller. It touches only fields fixed at construction (sdkClientFactory, +// sdkConfig, sdkInitTimeout, globalLoggers), so it needs no lock — mirroring startSDKClient, which also +// builds before locking. +// +// Returns the initialized client, or nil if the build failed, in which case it has already closed any +// half-built client and logged a structured error. initErr is deliberately left untouched on failure: +// it feeds the request middleware, and setting it to the new anchor's ErrInitializationFailed would 401 +// an env that is serving fine on the previous anchor. +func (c *envContextImpl) buildNewAnchorClient(newAnchor, previousAnchor config.SDKKey) sdks.LDClientContext { + client, err := c.sdkClientFactory(newAnchor, c.sdkConfig, c.sdkInitTimeout) + if err != nil || client == nil || !client.Initialized() { + var initialized bool + if client != nil { + initialized = client.Initialized() + _ = client.Close() + } + c.globalLoggers.Errorf("Re-anchor to SDK key %s failed (err=%v initialized=%v); "+ + "preserving previous anchor %s", + newAnchor.Masked(), err, initialized, previousAnchor.Masked()) + return nil + } + return client +} + +// commitReanchor is the second half of the re-anchor sequence: atomically move the rotator's anchor +// pointer, clear any stale init error now that a healthy client is current, and repoint downstream +// event/metrics forwarding. The caller must hold c.mu — reanchor holds it across the whole sequence, so +// the commit and Close() (which also takes c.mu) are mutually exclusive and Close can't tear the client +// or dispatcher out mid-commit. This mirrors addCredential, which likewise repoints event forwarding +// and reads rotator state under c.mu. +// +// Returns false without committing if the env was closed first, so callers report the rollback rather +// than a phantom success. +func (c *envContextImpl) commitReanchor(newAnchor, previousAnchor config.SDKKey, why string) bool { + if c.closed { + // Close() ran before we could commit. Don't flip the anchor or touch the (now-closed) dispatcher + // and metrics publisher; the env is being torn down. + return false + } + + c.keyRotator.CommitAnchor(newAnchor) + // A new anchor client is now authoritative, so any startSDKClient build still in flight from before + // this commit is stale: bump the generation so it discards itself instead of clobbering this client. + c.anchorClientGen++ + // The anchor now points at a healthy client (freshly built and Initialized, or a reused live + // client), so clear any init error a prior client left behind — otherwise GetInitError() and the + // request middleware would keep reporting a still-serving env as failed. + c.initErr = nil + + if c.metricsEventPub != nil { + c.metricsEventPub.ReplaceCredential(newAnchor) + } + if c.eventDispatcher != nil { + c.eventDispatcher.ReplaceCredential(newAnchor) + } + + // Big-segment synchronization is intentionally left pointing at the previous anchor key across a + // re-anchor: this matches pre-concurrent-keys behavior (there was no re-anchor, so it never moved) + // and does not regress. When big-segment re-anchor is implemented, its re-wire hook belongs right + // here, after the event/metrics ReplaceCredential calls — either recreate the BigSegmentSynchronizer + // for newAnchor, or add a credential-replacement method to it. + + c.globalLoggers.Infof("Re-anchored SDK from %s to %s (%s)", previousAnchor.Masked(), newAnchor.Masked(), why) + return true +} + +// rebuildEvaluator constructs the environment's Evaluator against the current data store. It is called +// after (re)creating an SDK client, once the store is available, and is shared by the initial client +// startup and the re-anchor path. It reads and writes envContextImpl fields directly, so the caller +// must hold c.mu. +// +// EnableSecondaryKey is set because we may evaluate for client-side SDKs sending old-style user data +// with the "secondary" attribute; it has no effect for newer SDKs that send contexts. +func (c *envContextImpl) rebuildEvaluator() { + store := c.storeAdapter.GetStore() + dataProvider := ldstoreimpl.NewDataStoreEvaluatorDataProvider(store, c.loggers) + evalOptions := []ldeval.EvaluatorOption{ + ldeval.EvaluatorOptionEnableSecondaryKey(true), + } + if c.sdkBigSegments != nil { + evalOptions = append(evalOptions, ldeval.EvaluatorOptionBigSegmentProvider(c.sdkBigSegments)) + } + c.evaluator = ldeval.NewEvaluatorWithOptions(dataProvider, evalOptions...) +} + +// registerCredentialMappings wires relay's downstream-facing routing for cred: it registers the +// credential with the env's stream machinery, builds the per-stream-provider HTTP handlers, and adds +// the connection→env mapping, so incoming SDK/client connections that authenticate with cred are +// served by this env. It does NOT start the upstream SDK client or repoint event/metrics forwarding — +// those are anchor-only concerns owned by the callers (addCredential, and the re-anchor sequence). +// The caller must hold c.mu. +func (c *envContextImpl) registerCredentialMappings(cred credential.SDKCredential) { + c.envStreams.AddCredential(cred) + for streamProvider, handlers := range c.handlers { + if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, cred)); h != nil { + handlers[cred] = h + } + } + c.connectionMapper.AddConnectionMapping(sdkauth.NewScoped(c.filterKey, cred), c) +} + +// triggerCredentialChanges drains the rotator's StepTime queue and applies the resulting additions +// and expirations. It runs on the cleanup ticker (cleanupExpiredCredentials), so it can fire at any +// moment — including while a synchronous re-anchor is in flight inside reconcileCredentials. +// +// It takes reconcileMu for the whole StepTime + add/remove pass so the ticker is serialized against +// reconcileCredentials exactly the way concurrent reconciles already are. Without it, a credential +// expiry firing during an in-flight re-anchor would drain the same StepTime queue the reconcile +// relies on (the ticker could steal additions a reconcile just queued) and could removeCredential — +// closing a client — partway through the re-anchor sequence. reconcileCredentials never calls this, +// so taking reconcileMu here introduces no re-entrancy. func (c *envContextImpl) triggerCredentialChanges(now time.Time) { c.reconcileMu.Lock() defer c.reconcileMu.Unlock() - c.drainCredentialChanges(now) -} -// drainCredentialChanges applies the rotator's queued additions (before expirations, so the accepted -// set is a superset during the transition). The caller must hold reconcileMu. -func (c *envContextImpl) drainCredentialChanges(now time.Time) { additions, expirations := c.keyRotator.StepTime(now) for _, cred := range additions { c.addCredential(cred) diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index e77b7eaa..ebd6a12a 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -60,12 +60,17 @@ func requireClientReady(t *testing.T, clientCh chan *testclient.FakeLDClient) *t func makeBasicEnv(t *testing.T, envConfig config.EnvConfig, clientFactory sdks.ClientFactoryFunc, loggers ldlog.Loggers, readyCh chan EnvContext) EnvContext { + return makeBasicEnvWithMapper(t, envConfig, clientFactory, loggers, readyCh, mockConnectionMapper{}) +} + +func makeBasicEnvWithMapper(t *testing.T, envConfig config.EnvConfig, clientFactory sdks.ClientFactoryFunc, + loggers ldlog.Loggers, readyCh chan EnvContext, connMapper ConnectionMapper) EnvContext { env, err := NewEnvContext(EnvContextImplParams{ Identifiers: EnvIdentifiers{ConfiguredName: envName}, EnvConfig: envConfig, ClientFactory: clientFactory, Loggers: loggers, - ConnectionMapper: mockConnectionMapper{}, + ConnectionMapper: connMapper, }, readyCh) require.NoError(t, err) return env @@ -81,6 +86,34 @@ func (m mockConnectionMapper) RemoveConnectionMapping(scopedCredential sdkauth.S } +// recordingConnectionMapper tracks which scoped credentials currently have a connection mapping, so a +// test can assert that a credential's mapping survived (or was torn down). +type recordingConnectionMapper struct { + mu sync.Mutex + mapped map[sdkauth.ScopedCredential]bool +} + +func (m *recordingConnectionMapper) AddConnectionMapping(scopedCredential sdkauth.ScopedCredential, _ EnvContext) { + m.mu.Lock() + defer m.mu.Unlock() + if m.mapped == nil { + m.mapped = map[sdkauth.ScopedCredential]bool{} + } + m.mapped[scopedCredential] = true +} + +func (m *recordingConnectionMapper) RemoveConnectionMapping(scopedCredential sdkauth.ScopedCredential) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.mapped, scopedCredential) +} + +func (m *recordingConnectionMapper) isMapped(filterKey config.FilterKey, cred credential.SDKCredential) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.mapped[sdkauth.NewScoped(filterKey, cred)] +} + func TestConstructorBasicProperties(t *testing.T) { envConfig := st.EnvWithAllCredentials.Config envConfig.TTL = configtypes.NewOptDuration(time.Hour) diff --git a/internal/relayenv/env_context_reanchor_double_test.go b/internal/relayenv/env_context_reanchor_double_test.go new file mode 100644 index 00000000..ec4b638b --- /dev/null +++ b/internal/relayenv/env_context_reanchor_double_test.go @@ -0,0 +1,110 @@ +package relayenv + +// Regression (multi-agent review, PR #739): a stale initial build must not clobber initErr after an +// A->B->A re-anchor. +// +// The initial startSDKClient(A) hangs, the anchor rotates A->B (healthy, initErr=nil), then B->A (a +// fresh healthy A client is built and committed, initErr=nil). When the ORIGINAL hung A build finally +// returns ErrInitializationFailed, it is no longer the client backing the anchor, so it must leave +// initErr untouched -- otherwise the middleware would 401 a healthy env. A gate of key-equality alone +// (sdkKey == AnchorKey()) is not enough here; the fix also requires this build to be the anchor's +// installed client. + +import ( + "sync/atomic" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" + st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + ld "github.com/launchdarkly/go-server-sdk/v7" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReanchor_DoubleReanchorBackDoesNotClobberInitErr(t *testing.T) { + envConfig := st.EnvMain.Config + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + healthy := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + var aCalls int32 + gate := make(chan struct{}) // releases the hung initial A build + enteredFirstA := make(chan struct{}, 1) + + factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == envConfig.SDKKey { // anchor A + if atomic.AddInt32(&aCalls, 1) == 1 { + // The ORIGINAL initial build: hang, then fail late -- returning a NON-NIL, uninitialized + // client with the error, as the real SDK does. If it were installed it would close the + // fresh healthy A client from the B->A rebuild and swap in this dead one. + enteredFirstA <- struct{}{} + <-gate + return &testclient.FakeLDClient{Key: sdkKey, CloseCh: make(chan struct{})}, ld.ErrInitializationFailed + } + // The B->A rebuild: healthy. + return healthy(sdkKey, cfg, timeout) + } + return healthy(sdkKey, cfg, timeout) // B healthy + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) + defer env.Close() + envImpl := env.(*envContextImpl) + + <-enteredFirstA // initial A build is hung; anchor is A, no client installed yet + + now := time.Unix(2000, 0) + expiry := now.Add(time.Hour) + + // A -> B (B brand new/healthy; A grace-demoted). + setB, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: reanchorSyncTestKey2}). + WithSDKKey(credential.SDKKeyParams{Value: envConfig.SDKKey, Expiry: &expiry}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: envConfig.MobileKey}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(setB, now) + + // B -> A (A still in grace, no client -> reanchor builds a fresh healthy A client and commits). + setA, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). + WithSDKKey(credential.SDKKeyParams{Value: reanchorSyncTestKey2, Expiry: &expiry}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: envConfig.MobileKey}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(setA, now) + + // Sanity: A is the anchor again, on a fresh healthy client, env healthy. + require.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey()) + require.NoError(t, env.GetInitError(), "sanity: healthy after re-anchoring back to A") + freshA := env.GetClient() + require.NotNil(t, freshA) + require.True(t, freshA.Initialized()) + + // The ORIGINAL hung A build now fails late. + close(gate) + <-readyCh + + // It must NOT clobber the healthy env's initErr. + assert.NoError(t, env.GetInitError(), + "a stale initial build failure must not clobber a healthy re-anchored-back env") + assert.NotEqual(t, ld.ErrInitializationFailed, env.GetInitError(), + "the middleware would 401 the whole healthy env on ErrInitializationFailed") + // The fresh healthy client from the B->A rebuild must still be the anchor's client -- the stale + // build must not have closed it and swapped in its dead, uninitialized client. + assert.Same(t, freshA, env.GetClient(), "the stale build must not replace the fresh healthy client") + assert.True(t, env.GetClient().Initialized(), "the anchor client is still the initialized one") +} diff --git a/internal/relayenv/env_context_reanchor_rollback_test.go b/internal/relayenv/env_context_reanchor_rollback_test.go new file mode 100644 index 00000000..6337c3d3 --- /dev/null +++ b/internal/relayenv/env_context_reanchor_rollback_test.go @@ -0,0 +1,71 @@ +package relayenv + +// Regression: a grace-demotion re-anchor rollback must not leave the still-authoritative anchor exposed +// to the cleanup ticker. +// +// Scenario (the default backend rotation): anchor A is demoted with a +1h grace expiry while brand-new +// key B is designated the new anchor. B's client fails to initialize, so the synchronous re-anchor rolls +// back: A stays the anchor and keeps serving. A's accepted entry still carries the grace expiry and +// CommitAnchor never ran (A is still anchorKey), so the cleanup ticker firing past the grace window must +// NOT expire A and close the env's only client -- StepTime's anchor guard prevents that. Without the +// guard, GetClient() returns nil while the rotator still names A the anchor: a silent upstream outage. + +import ( + "errors" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" + st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + ld "github.com/launchdarkly/go-server-sdk/v7" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReanchorRollbackGraceDemotionKeepsAnchorServing(t *testing.T) { + envConfig := st.EnvMain.Config + fakeErr := errors.New("re-anchor: new client init refused") + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + healthyFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + // Fail only for the new anchor; the original anchor builds fine. + factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == reanchorSyncTestKey2 { + return nil, fakeErr + } + return healthyFactory(sdkKey, cfg, timeout) + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + originalClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == originalClient }, time.Second, 10*time.Millisecond) + + now := time.Unix(2000, 0) + // Re-anchor A->B with A grace-demoted (+1h). B's build fails, so this rolls back. + reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) + + // Rollback holds immediately: A is still the anchor and still serving. + require.Same(t, originalClient, env.GetClient(), "rollback keeps the old anchor serving") + require.Equal(t, envConfig.SDKKey, env.(*envContextImpl).keyRotator.AnchorKey(), "anchor stayed on the old key") + + // The cleanup ticker fires after the grace window. A is STILL the anchor, so it must keep serving -- + // a key's grace expiry must not apply to it while it is the authoritative anchor. + env.(*envContextImpl).triggerCredentialChanges(now.Add(time.Hour + time.Minute)) + + assert.Equal(t, envConfig.SDKKey, env.(*envContextImpl).keyRotator.AnchorKey(), + "the anchor pointer is unchanged after the ticker") + assert.NotNil(t, env.GetClient(), + "the cleanup ticker must not reap the still-authoritative anchor's client (env would go dark)") +} diff --git a/internal/relayenv/env_context_reanchor_synchronous_test.go b/internal/relayenv/env_context_reanchor_synchronous_test.go new file mode 100644 index 00000000..9c0ca5fc --- /dev/null +++ b/internal/relayenv/env_context_reanchor_synchronous_test.go @@ -0,0 +1,728 @@ +package relayenv + +// Regression tests for the synchronous re-anchor sequence. +// +// These cover: re-anchoring to a new key (build success and init-failure rollback), re-anchoring to a +// previously-accepted key (reuse the existing client), no orphan clients, store-handover survival, and +// the mobile-primary repoint signal (the gap when the new primary mobile key was already in the +// accepted set, so the primary-mobile gate does not fire for it). + +import ( + "errors" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" + st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + ld "github.com/launchdarkly/go-server-sdk/v7" + "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const reanchorSyncTestKey2 = config.SDKKey("reanchor-sync-new-anchor") +const reanchorSyncExpiringKey = config.SDKKey("reanchor-sync-expiring-key") + +// reanchorViaReconcile drives the production ReconcileCredentials path with a payload that +// designates newKey as the anchor and keeps oldKey accepted with an expiry one hour in the future, +// mirroring the backend's default-rotation behavior (the new anchor is non-expiring; the demoted +// old anchor carries an expiry). extraAcceptedSDK, if defined, is added as a permanent non-anchor +// SDK key — used to set up the reuse path (a key already in the accepted set later becoming the anchor). +func reanchorViaReconcile( + t *testing.T, + env EnvContext, + newKey, oldKey, extraAcceptedSDK config.SDKKey, + primaryMobile config.MobileKey, + envID config.EnvironmentID, + now time.Time, +) { + t.Helper() + builder := credential.NewAcceptedSetBuilder().WithAnchor(credential.SDKKeyParams{Value: newKey}) + if oldKey.Defined() && oldKey != newKey { + expiry := now.Add(time.Hour) + builder = builder.WithSDKKey(credential.SDKKeyParams{Value: oldKey, Expiry: &expiry}) + } + if extraAcceptedSDK.Defined() && extraAcceptedSDK != newKey { + builder = builder.WithSDKKey(credential.SDKKeyParams{Value: extraAcceptedSDK}) + } + if primaryMobile.Defined() { + builder = builder.WithPrimaryMobileKey(credential.MobileKeyParams{Value: primaryMobile}) + } + if envID.Defined() { + builder = builder.WithEnvironmentID(envID) + } + set, err := builder.Build() + require.NoError(t, err) + env.(*envContextImpl).reconcileCredentials(set, now) +} + +// TestReanchorSync_CaseA_BuildsNewClientAndMovesAnchor exercises the happy path where the new anchor +// is a brand-new SDK key with no existing client: the build succeeds, and the anchor commits. The +// store is handed over (no empty-store window) and the new client becomes current. +func TestReanchorSync_CaseA_BuildsNewClientAndMovesAnchor(t *testing.T) { + featureKind := ldstoreimpl.Features() + flagKey := st.Flag1ServerSide.Flag.Key + envConfig := st.EnvMain.Config + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactoryWithChannel(true, clientCh), mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + originalClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == originalClient }, time.Second, 10*time.Millisecond) + + // Populate the store as the original anchor's stream sync would. + require.NoError(t, env.GetStore().Init(st.AllData)) + originalStore := env.GetStore() + + // Re-anchor onto a brand-new key via ReconcileCredentials. + now := time.Unix(2000, 0) + reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) + + // A new client was built synchronously and is now the current anchor's client. + newClient := requireClientReady(t, clientCh) + assert.NotSame(t, originalClient, newClient, "a new anchor key builds a fresh client") + assert.Same(t, newClient, env.GetClient(), "GetClient returns the new anchor's client after the commit") + assert.Nil(t, env.GetInitError(), "successful re-anchor clears any prior init error") + + // The rotator's primary now names the new key (CommitAnchor ran). + assert.Equal(t, reanchorSyncTestKey2, env.(*envContextImpl).keyRotator.AnchorKey()) + assert.Contains(t, env.GetCredentials(), credential.SDKCredential(reanchorSyncTestKey2)) + + // Store handover: the wrapper is the same instance, still initialized, data intact. + assert.Same(t, originalStore, env.GetStore(), "store handover: the wrapper survives the re-anchor") + got, err := env.GetStore().Get(featureKind, flagKey) + require.NoError(t, err) + assert.NotNil(t, got.Item, "data survives the re-anchor (no empty-store window)") + + // The old client is still alive — its grace period has not elapsed (the old client keeps serving; + // closure happens via removeCredential when the expiry fires). + envImpl := env.(*envContextImpl) + envImpl.mu.RLock() + _, oldStillPresent := envImpl.clients[envConfig.SDKKey] + envImpl.mu.RUnlock() + assert.True(t, oldStillPresent, "old anchor's client retained during its grace period") +} + +// TestReanchorSync_CaseA_InitFailureRollsBack confirms that a failed new-client init does NOT move +// the rotator's anchor, does NOT install the broken client, and does NOT close the previous client. +// The old anchor keeps serving; the failure surfaces as a structured Errorf log (initErr is left +// untouched so the still-healthy env is not marked failed). This is the all-or-nothing atomicity +// requirement applied to re-anchor. +func TestReanchorSync_CaseA_InitFailureRollsBack(t *testing.T) { + envConfig := st.EnvMain.Config + fakeErr := errors.New("re-anchor: new client init refused") + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + healthyFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + // Fail only for the new anchor key; succeed for the original. + factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == reanchorSyncTestKey2 { + return nil, fakeErr + } + return healthyFactory(sdkKey, cfg, timeout) + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + originalClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == originalClient }, time.Second, 10*time.Millisecond) + + now := time.Unix(2000, 0) + reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) + + // Rollback: the env stays healthy on the previous anchor, so GetInitError stays nil — setting it + // would 401 a still-serving env at the request middleware. The failure surfaces via a structured + // Error log instead. Anchor pointer unchanged, no new client installed, old client still in place. + assert.NoError(t, env.GetInitError(), "a failed re-anchor must not mark the still-serving env as failed") + mockLog.AssertMessageMatch(t, true, ldlog.Error, "Re-anchor to SDK key .* failed") + assert.Same(t, originalClient, env.GetClient(), "GetClient still returns the previous anchor's client") + assert.Equal(t, envConfig.SDKKey, env.(*envContextImpl).keyRotator.AnchorKey(), "anchor pointer stayed on the previous key") + + envImpl := env.(*envContextImpl) + envImpl.mu.RLock() + _, newAnchorClientInstalled := envImpl.clients[reanchorSyncTestKey2] + _, oldStillPresent := envImpl.clients[envConfig.SDKKey] + envImpl.mu.RUnlock() + assert.False(t, newAnchorClientInstalled, "no client installed for the failed new anchor") + assert.True(t, oldStillPresent, "old anchor's client preserved on rollback") +} + +// TestReanchorSync_CaseB_ReusesExistingClient covers re-anchoring onto a key that already has a +// live client. The simplest deterministic setup: re-anchor A→B (B's client built), then re-anchor +// B→A while A is still in its grace period. A's client still exists, so the second re-anchor must +// reuse it and build nothing new. +func TestReanchorSync_CaseB_ReusesExistingClient(t *testing.T) { + envConfig := st.EnvMain.Config + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactoryWithChannel(true, clientCh), mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + originalClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == originalClient }, time.Second, 10*time.Millisecond) + require.NoError(t, env.GetStore().Init(st.AllData)) + + // First re-anchor: original → key2 (new key; key2 client built). Keep the original in grace. + now := time.Unix(2000, 0) + reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) + key2Client := requireClientReady(t, clientCh) + require.Same(t, key2Client, env.GetClient()) + + // The original anchor's client is still alive in its grace period. + envImpl := env.(*envContextImpl) + envImpl.mu.RLock() + originalStillPresent := envImpl.clients[envConfig.SDKKey] == originalClient + envImpl.mu.RUnlock() + require.True(t, originalStillPresent, "original client retained for reuse") + + // Second re-anchor: key2 → original. The original's client exists, so this is the reuse path: no + // Build, the existing client is reused, the anchor flips, and ReplaceCredential runs. + reanchorViaReconcile(t, env, envConfig.SDKKey, reanchorSyncTestKey2, "", envConfig.MobileKey, envConfig.EnvID, now) + + // No new client was created — clientCh must be empty (every prior client was drained). + select { + case c := <-clientCh: + t.Fatalf("re-anchoring to a key with an existing client must not build a new one, but one was created: %v", c.Key) + case <-time.After(100 * time.Millisecond): + } + + assert.Same(t, originalClient, env.GetClient(), "reuses the existing client for the re-anchored key") + assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "anchor flipped back to the original key") +} + +// TestReanchorSync_CredentialExpiryDuringReanchorIsSerialized exercises the concurrency gap closed +// by serializing the cleanup ticker against reconcileMu: a credential expiry firing (via the ticker +// path, triggerCredentialChanges) while a synchronous re-anchor is mid-build must not run its +// StepTime + add/remove pass concurrently with the in-flight re-anchor. Both paths drain the same +// StepTime queue and can close clients, so they must be serialized the way concurrent reconciles are. +// +// The test wedges a re-anchor open by blocking the new anchor's client build, then fires the ticker +// from another goroutine and asserts (a) the ticker is blocked while the re-anchor holds reconcileMu, +// (b) it completes once the re-anchor releases it, and (c) the final state is consistent — the new +// anchor committed, the expiring non-anchor key dropped, the demoted old anchor retained in grace. +func TestReanchorSync_CredentialExpiryDuringReanchorIsSerialized(t *testing.T) { + envConfig := st.EnvMain.Config + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + now := time.Unix(2000, 0) + expiringExpiry := now.Add(30 * time.Minute) // the non-anchor key the ticker will drop + graceExpiry := now.Add(2 * time.Hour) // the demoted old anchor stays alive in its grace period + tickerTime := now.Add(time.Hour) // between the two expiries: drops only the expiring key + + // The new anchor's client build blocks until releaseBuild is closed, holding the re-anchor (and + // thus reconcileMu) open so the ticker has a window to (try to) run concurrently. + buildEntered := make(chan struct{}) + releaseBuild := make(chan struct{}) + + clientCh := make(chan *testclient.FakeLDClient, 10) + healthy := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == reanchorSyncTestKey2 { + close(buildEntered) + <-releaseBuild + } + return healthy(sdkKey, cfg, timeout) + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + originalClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == originalClient }, time.Second, 10*time.Millisecond) + require.NoError(t, env.GetStore().Init(st.AllData)) + + envImpl := env.(*envContextImpl) + + // Accept a second SDK key as a non-anchor server key carrying a future expiry. addCredential's + // anchor gate won't build a client for it, so it has no client of its own. + initialSet, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). + WithSDKKey(credential.SDKKeyParams{Value: reanchorSyncExpiringKey, Expiry: &expiringExpiry}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: envConfig.MobileKey}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(initialSet, now) + require.Contains(t, env.GetCredentials(), credential.SDKCredential(reanchorSyncExpiringKey)) + + // Re-anchor onto a brand-new key while keeping both the demoted old anchor and the expiring key + // accepted. The build will block, holding reconcileMu. + reanchorSet, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: reanchorSyncTestKey2}). + WithSDKKey(credential.SDKKeyParams{Value: envConfig.SDKKey, Expiry: &graceExpiry}). + WithSDKKey(credential.SDKKeyParams{Value: reanchorSyncExpiringKey, Expiry: &expiringExpiry}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: envConfig.MobileKey}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + + reconcileDone := make(chan struct{}) + go func() { + defer close(reconcileDone) + envImpl.reconcileCredentials(reanchorSet, now) + }() + + // Wait until the re-anchor is wedged open inside the client build (holding reconcileMu). + <-buildEntered + + // Fire the cleanup ticker's work while the re-anchor holds reconcileMu. With the ticker serialized + // against reconcileMu, this must block until the re-anchor releases it. + tickerDone := make(chan struct{}) + go func() { + defer close(tickerDone) + envImpl.triggerCredentialChanges(tickerTime) + }() + + select { + case <-tickerDone: + t.Fatal("the cleanup ticker ran its StepTime + add/remove pass concurrently with an in-flight " + + "re-anchor; reconcileMu did not serialize the ticker against reconcileCredentials") + case <-time.After(100 * time.Millisecond): + // Expected: the ticker is blocked on reconcileMu. + } + + // Release the build; the re-anchor finishes and drops reconcileMu, unblocking the ticker. + close(releaseBuild) + <-reconcileDone + select { + case <-tickerDone: + case <-time.After(time.Second): + t.Fatal("the cleanup ticker did not complete after the re-anchor released reconcileMu") + } + + // Final state is consistent: the new anchor committed and serves its client, the expiring + // non-anchor key was dropped by the ticker, and the demoted old anchor is retained in grace. + newClient := requireClientReady(t, clientCh) + assert.Same(t, newClient, env.GetClient(), "the new anchor's client is current after the re-anchor") + assert.Equal(t, reanchorSyncTestKey2, envImpl.keyRotator.AnchorKey()) + assert.NotContains(t, env.GetCredentials(), credential.SDKCredential(reanchorSyncExpiringKey), + "the expiring non-anchor key was dropped by the ticker") + + envImpl.mu.RLock() + _, oldStillPresent := envImpl.clients[envConfig.SDKKey] + envImpl.mu.RUnlock() + assert.True(t, oldStillPresent, "demoted old anchor's client retained during its grace period") +} + +// TestReanchorSync_MobilePrimaryRepoint_AlreadyAcceptedKey covers the primary-mobile gate's gap: +// when the primary mobile key switches to a key that was ALREADY in the accepted set, that key is +// not in StepTime's additions, so addCredential's gate never fires for it. reconcileCredentials +// must therefore call eventDispatcher.ReplaceCredential synchronously via MobilePrimaryRepoint. +// +// This test asserts the rotator-level signal: after reconciling to make an already-accepted mobile +// key the primary, the env's primary mobile key reflects the change. (The ReplaceCredential side +// effect on the dispatcher is exercised by the events package's own dispatcher tests; here we +// confirm the env drives the primary-mobile transition correctly without spawning a client.) +func TestReanchorSync_MobilePrimaryRepoint_AlreadyAcceptedKey(t *testing.T) { + envConfig := st.EnvMain.Config + mob1 := config.MobileKey("mob-primary-1") + mob2 := config.MobileKey("mob-primary-2") + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactoryWithChannel(true, clientCh), mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + _ = requireClientReady(t, clientCh) + + now := time.Unix(2000, 0) + envImpl := env.(*envContextImpl) + + // Accept both mobile keys, mob1 primary. mob2 is accepted but not primary. + set1, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: mob1}). + WithMobileKey(credential.MobileKeyParams{Value: mob2}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(set1, now) + require.Equal(t, mob1, envImpl.keyRotator.MobileKey()) + require.Contains(t, env.GetCredentials(), credential.SDKCredential(mob2)) + + // Now make mob2 (already accepted) the primary mobile key. No SDK anchor change here. + set2, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: mob2}). + WithMobileKey(credential.MobileKeyParams{Value: mob1}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(set2, now) + + assert.Equal(t, mob2, envImpl.keyRotator.MobileKey(), "primary mobile key repointed to the already-accepted key") + // No SDK client should have been spawned by a mobile-only change. + select { + case c := <-clientCh: + t.Fatalf("a mobile-primary repoint must not build an SDK client, got: %v", c.Key) + case <-time.After(100 * time.Millisecond): + } +} + +// TestReanchorSync_PreviouslyAcceptedNonAnchorPromotedToAnchor covers the case the two-signal split in +// reanchor exists for: a server SDK key accepted as a NON-anchor has its credential mappings registered +// by addCredential but no client (only the anchor gets one). Promoting it to anchor must NOT re-register +// its mappings (NewAnchorPreviouslyAccepted == true) but MUST build a client (existingClient == nil). +func TestReanchorSync_PreviouslyAcceptedNonAnchorPromotedToAnchor(t *testing.T) { + envConfig := st.EnvMain.Config + nonAnchorKey := config.SDKKey("reanchor-sync-nonanchor") + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactoryWithChannel(true, clientCh), mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + originalClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == originalClient }, time.Second, 10*time.Millisecond) + require.NoError(t, env.GetStore().Init(st.AllData)) + + now := time.Unix(2000, 0) + envImpl := env.(*envContextImpl) + + // Accept a second server SDK key as a NON-anchor (permanent). Its mappings are registered but no + // client is built — only the anchor owns an upstream client. + set1, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). + WithSDKKey(credential.SDKKeyParams{Value: nonAnchorKey}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: envConfig.MobileKey}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(set1, now) + require.Contains(t, env.GetCredentials(), credential.SDKCredential(nonAnchorKey)) + select { + case c := <-clientCh: + t.Fatalf("a non-anchor server key must not build a client, got: %v", c.Key) + case <-time.After(100 * time.Millisecond): + } + envImpl.mu.RLock() + _, nonAnchorHasClient := envImpl.clients[nonAnchorKey] + envImpl.mu.RUnlock() + require.False(t, nonAnchorHasClient, "non-anchor key has no client of its own") + + // Promote the previously-accepted non-anchor key to anchor: mappings already exist (skip + // re-registration), but there is no client, so the re-anchor must build one. + reanchorViaReconcile(t, env, nonAnchorKey, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) + + newClient := requireClientReady(t, clientCh) + assert.NotSame(t, originalClient, newClient, "promoting a client-less accepted key builds a fresh client") + assert.Same(t, newClient, env.GetClient()) + assert.Equal(t, nonAnchorKey, envImpl.keyRotator.AnchorKey(), "anchor committed to the promoted key") + assert.NoError(t, env.GetInitError()) +} + +// TestReanchorSync_Offline_CommitsWithoutBuildingClient covers the offline re-anchor branch: when the +// env is offline, re-anchoring to a new key must commit the anchor WITHOUT building a new upstream +// client. (The initial anchor client is still created at startup; offline only skips the re-anchor +// build.) +func TestReanchorSync_Offline_CommitsWithoutBuildingClient(t *testing.T) { + envConfig := st.EnvMain.Config + envConfig.Offline = true + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactoryWithChannel(true, clientCh), mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + _ = requireClientReady(t, clientCh) // drain the initial anchor client + + now := time.Unix(2000, 0) + reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) + + // The anchor commits, but the offline branch builds no new client. + assert.Equal(t, reanchorSyncTestKey2, env.(*envContextImpl).keyRotator.AnchorKey(), "offline re-anchor commits the anchor") + select { + case c := <-clientCh: + t.Fatalf("an offline re-anchor must not build a new SDK client, got: %v", c.Key) + case <-time.After(100 * time.Millisecond): + } + assert.NoError(t, env.GetInitError()) +} + +// TestReanchorSync_RollbackWithImmediateRevocationKeepsOldAnchorServing covers the edge where a +// reconcile both moves the anchor to a new key AND immediately revokes the current anchor (no grace +// expiry), and the new anchor's client fails to build. The re-anchor rolls back, backing out just the +// anchor change: the previous anchor is kept serving and re-admitted to the accepted set, and the +// failed new key is dropped. +func TestReanchorSync_RollbackWithImmediateRevocationKeepsOldAnchorServing(t *testing.T) { + envConfig := st.EnvMain.Config + fakeErr := errors.New("re-anchor: new client init refused") + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + healthyFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == reanchorSyncTestKey2 { + return nil, fakeErr + } + return healthyFactory(sdkKey, cfg, timeout) + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + originalClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == originalClient }, time.Second, 10*time.Millisecond) + require.NoError(t, env.GetStore().Init(st.AllData)) + + envImpl := env.(*envContextImpl) + now := time.Unix(2000, 0) + + // Re-anchor to a brand-new key that fails to init, while immediately revoking the current anchor — + // it is omitted from the payload entirely (no grace expiry). + set, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: reanchorSyncTestKey2}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: envConfig.MobileKey}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(set, now) + + // The rollback backed out the anchor change: the previous anchor still serves and is still accepted, + // the failed new key is gone, and the env is not marked failed. + assert.NoError(t, env.GetInitError(), "a rolled-back re-anchor must not mark the env failed") + assert.Same(t, originalClient, env.GetClient(), "the previous anchor's client keeps serving despite the revocation") + assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "anchor pointer stayed on the previous key") + assert.Contains(t, env.GetCredentials(), credential.SDKCredential(envConfig.SDKKey), "previous anchor re-admitted to the accepted set") + assert.NotContains(t, env.GetCredentials(), credential.SDKCredential(reanchorSyncTestKey2), "failed new anchor dropped from the accepted set") + + envImpl.mu.RLock() + _, newHasClient := envImpl.clients[reanchorSyncTestKey2] + _, oldHasClient := envImpl.clients[envConfig.SDKKey] + envImpl.mu.RUnlock() + assert.False(t, newHasClient, "no client for the failed new anchor") + assert.True(t, oldHasClient, "previous anchor's client retained") + + // A later cleanup ticker must not reap the re-admitted anchor: RevertAnchorChange re-admits it as a + // permanent (nil-expiry) key, so there is nothing to expire. (Complements the grace-demotion case, + // where the anchor keeps a stale expiry and the StepTime guard is what protects it.) + envImpl.triggerCredentialChanges(now.Add(2 * time.Hour)) + assert.Same(t, originalClient, env.GetClient(), "the re-admitted anchor still serves after a later ticker") + assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey()) +} + +// TestReanchorSync_PreviouslyAcceptedAnchorPromotionFailureKeepsItsMappings covers a failed promotion +// of an already-accepted non-anchor key: the rollback must NOT tear down that key's credential +// mappings (they predate this reconcile), since RevertAnchorChange keeps it accepted. It should revert +// cleanly to the non-anchor key it already was. +func TestReanchorSync_PreviouslyAcceptedAnchorPromotionFailureKeepsItsMappings(t *testing.T) { + envConfig := st.EnvMain.Config + nonAnchorKey := config.SDKKey("reanchor-sync-nonanchor-promote-fail") + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + healthyFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + // Fail the build only when nonAnchorKey is promoted to anchor. + factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == nonAnchorKey { + return nil, errors.New("promotion build refused") + } + return healthyFactory(sdkKey, cfg, timeout) + } + + mapper := &recordingConnectionMapper{} + readyCh := make(chan EnvContext, 1) + env := makeBasicEnvWithMapper(t, envConfig, factory, mockLog.Loggers, readyCh, mapper) + defer env.Close() + + require.Equal(t, env, requireEnvReady(t, readyCh)) + originalClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == originalClient }, time.Second, 10*time.Millisecond) + require.NoError(t, env.GetStore().Init(st.AllData)) + + envImpl := env.(*envContextImpl) + now := time.Unix(2000, 0) + + // Accept nonAnchorKey as a non-anchor server key: mappings are registered (and it becomes mapped), + // but no client is built for it. + set1, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). + WithSDKKey(credential.SDKKeyParams{Value: nonAnchorKey}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: envConfig.MobileKey}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(set1, now) + require.True(t, mapper.isMapped(envConfig.FilterKey, nonAnchorKey), "non-anchor key is mapped once accepted") + + // Promote nonAnchorKey to anchor; its client build fails, so the re-anchor rolls back. + reanchorViaReconcile(t, env, nonAnchorKey, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) + + // The failed promotion rolled back: nonAnchorKey stays accepted AND keeps its connection mapping — + // the rollback must not strip mappings that existed before this reconcile. + assert.Contains(t, env.GetCredentials(), credential.SDKCredential(nonAnchorKey), "previously-accepted key stays accepted after a failed promotion") + assert.True(t, mapper.isMapped(envConfig.FilterKey, nonAnchorKey), "its connection mapping must survive the rollback") + // The previous anchor still serves. + assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey()) + assert.Same(t, originalClient, env.GetClient()) + assert.NoError(t, env.GetInitError()) +} + +// TestReanchor_SupersededFailingBuildDoesNotClobberInitErr: the initial startSDKClient(A) is still in +// flight when a re-anchor to healthy B commits (initErr=nil). A's build then fails -- returning a +// NON-NIL, uninitialized client with the error, exactly as the real SDK's MakeCustomClient does. Because +// a re-anchor committed since this build launched, it is superseded: it must be discarded, not installed, +// and must not touch initErr -- otherwise it would clobber a healthy env into a whole-env 401. +func TestReanchor_SupersededFailingBuildDoesNotClobberInitErr(t *testing.T) { + envConfig := st.EnvMain.Config + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + healthy := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + gate := make(chan struct{}) + entered := make(chan struct{}, 1) + factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == envConfig.SDKKey { + // The ORIGINAL anchor A: block, then fail like the real SDK -- a non-nil, uninitialized client + // plus the error (MakeCustomClient returns the client on init failure/timeout). + entered <- struct{}{} + <-gate + return &testclient.FakeLDClient{Key: sdkKey, CloseCh: make(chan struct{})}, ld.ErrInitializationFailed + } + return healthy(sdkKey, cfg, timeout) + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) + defer env.Close() + + envImpl := env.(*envContextImpl) + <-entered // A's initial build is blocked; anchor is still A. + + // Re-anchor A -> B (B brand new/healthy; A grace-demoted +1h). B builds, commits, clears initErr. + now := time.Unix(2000, 0) + expiry := now.Add(time.Hour) + set, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: reanchorSyncTestKey2}). + WithSDKKey(credential.SDKKeyParams{Value: envConfig.SDKKey, Expiry: &expiry}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: envConfig.MobileKey}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(set, now) + require.Equal(t, reanchorSyncTestKey2, envImpl.keyRotator.AnchorKey()) + require.NoError(t, env.GetInitError(), "sanity: healthy after re-anchor to B") + + // The stale initial A build now fails late. + close(gate) + <-readyCh + + assert.NoError(t, env.GetInitError(), + "a superseded build's late failure must not clobber the healthy env's initErr") + assert.NotEqual(t, ld.ErrInitializationFailed, env.GetInitError()) + assert.NotNil(t, env.GetClient(), "B's healthy client still serves") + // The superseded build was discarded, not installed for A. + envImpl.mu.RLock() + _, aHasClient := envImpl.clients[envConfig.SDKKey] + envImpl.mu.RUnlock() + assert.False(t, aHasClient, "the superseded build must not be installed for A") +} + +// TestReanchor_SupersededLateBuildIsDiscarded: even a *successful* initial build is discarded if a +// re-anchor committed a fresh anchor client while it was in flight. Installing it would tear down the +// current anchor client (startSDKClient's stale-client guard closes whatever is installed) and swap in +// an obsolete one. The superseded build must be dropped, leaving the committed anchor untouched. +func TestReanchor_SupersededLateBuildIsDiscarded(t *testing.T) { + envConfig := st.EnvMain.Config + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + healthy := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + gate := make(chan struct{}) + entered := make(chan struct{}, 1) + factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == envConfig.SDKKey { + // The ORIGINAL anchor A: block, then build SUCCESSFULLY (late). + entered <- struct{}{} + <-gate + } + return healthy(sdkKey, cfg, timeout) + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) + defer env.Close() + + envImpl := env.(*envContextImpl) + <-entered + + now := time.Unix(2000, 0) + expiry := now.Add(time.Hour) + set, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: reanchorSyncTestKey2}). + WithSDKKey(credential.SDKKeyParams{Value: envConfig.SDKKey, Expiry: &expiry}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: envConfig.MobileKey}). + WithEnvironmentID(envConfig.EnvID). + Build() + require.NoError(t, err) + envImpl.reconcileCredentials(set, now) + require.Equal(t, reanchorSyncTestKey2, envImpl.keyRotator.AnchorKey()) + bClient := env.GetClient() + require.NotNil(t, bClient) + + // A's initial build completes successfully, after the re-anchor -- but it is superseded. + close(gate) + <-readyCh + + // It was discarded: A has no client, and B's client is untouched. + envImpl.mu.RLock() + _, aHasClient := envImpl.clients[envConfig.SDKKey] + envImpl.mu.RUnlock() + assert.False(t, aHasClient, "a superseded successful build must be discarded, not installed for A") + assert.Same(t, bClient, env.GetClient(), "B's committed client must not be torn down by the stale build") + assert.Equal(t, reanchorSyncTestKey2, envImpl.keyRotator.AnchorKey(), "anchor stays B") + assert.NoError(t, env.GetInitError()) +} diff --git a/internal/relayenv/env_context_reanchor_test.go b/internal/relayenv/env_context_reanchor_test.go index d3009db3..22543955 100644 --- a/internal/relayenv/env_context_reanchor_test.go +++ b/internal/relayenv/env_context_reanchor_test.go @@ -116,9 +116,11 @@ func TestReanchorPoC_H1_SharedStoreAdapterRebuildSemantics(t *testing.T) { featureKind := ldstoreimpl.Features() flagKey := st.Flag1ServerSide.Flag.Key - // A second storeAdapter.Build hands back the SAME wrapper, with its data still in place, rather - // than constructing a fresh wrapper around a fresh (empty) underlying store. This is what lets a - // re-anchor's new client keep serving the populated store instead of starting empty. + // Original PoC finding: each storeAdapter.Build call constructed a fresh wrapper around a fresh + // underlying store, so the new anchor's client would start empty. The store-handover change + // (SSERelayDataStoreAdapter.Build reusing its existing wrapper, with refcounted Close) inverts + // this: the second client receives the SAME wrapper, with its data still in place. This sub-test + // now asserts the post-fix invariant. t.Run("in-memory factory reuses the existing store on a second client init (store handover)", func(t *testing.T) { rec := &recordingStreamUpdates{} adapter := store.NewSSERelayDataStoreAdapter(ldcomponents.InMemoryDataStore(), rec) @@ -411,11 +413,12 @@ func TestReanchorPoC_H4_HTTPConfigIsKeyIndependentExceptAuthHeader(t *testing.T) // Hypothesis 5: Order of operations / the in-memory store window. // ----------------------------------------------------------------------------------------------- -// TestReanchorPoC_H5_StoreSurvivesReAnchor asserts that a re-anchor keeps the env's in-memory data -// store instance intact: the same instance, still initialized, with its data preserved. This holds -// because SSERelayDataStoreAdapter.Build hands its existing wrapper over to the new anchor's client -// (with refcounted Close) rather than building a fresh, empty store. It is the end-to-end proof that -// store handover holds through env_context. +// TestReanchorPoC_H5_StoreSurvivesReAnchor was originally a proof-of-concept test asserting the +// *broken* pre-handover behavior: that re-anchor caused the env's in-memory store to be replaced with +// a fresh, empty one. Once SSERelayDataStoreAdapter.Build was changed to hand over the existing wrapper +// to the new client (refcounted Close), that breakage is gone. This now asserts the post-fix +// invariant — the data store instance survives the re-anchor and keeps its data — and is preserved +// as the executable proof that handover holds end-to-end through env_context. func TestReanchorPoC_H5_StoreSurvivesReAnchor(t *testing.T) { featureKind := ldstoreimpl.Features() flagKey := st.Flag1ServerSide.Flag.Key @@ -460,16 +463,18 @@ func TestReanchorPoC_H5_StoreSurvivesReAnchor(t *testing.T) { assert.NotNil(t, got2.Item, "data is preserved across re-anchor") } -// TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow exercises store handover at the store layer: -// because relay owns the data store implementation (it hands the SDK a single storeAdapter), the -// re-anchor hands the existing store to the new client instead of letting it build a fresh one. It is -// modeled here by a DataStoreFactory that returns the same underlying store on every Build, so the new -// anchor's client sees the populated, initialized store immediately with no empty-store window. +// TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow validates the reviewer suggestion that, because +// relay owns the data store implementation (it hands the SDK a single storeAdapter), the re-anchor can +// hand the existing store over to the new client instead of letting it build a fresh one. Modeled here +// by a DataStoreFactory that returns the same underlying store on every Build; the production change +// is to make SSERelayDataStoreAdapter reuse its store across the swap. With handover the new anchor's +// client sees the populated, initialized store immediately -- no empty-store window (contrast +// TestReanchorPoC_H5_InMemoryStoreIsWipedByReAnchor). // -// The store's lifecycle is owned by the adapter, not the client: streamUpdatesStoreWrapper.Close() -// closes the underlying store, so when the retiring and new clients share one underlying store the -// retiring client's Close() must not tear it down. The fake client cannot exercise the real client's -// Close(), so that half of the contract is covered by TestRealClient_HandoverPreservesUnderlyingStore. +// CAVEAT for the implementation (not reproducible with the fake client, so documented here): +// streamUpdatesStoreWrapper.Close() closes the underlying store. If the new client wraps the +// SAME underlying store, closing the retiring client must NOT close it -- the store's lifecycle has to +// be owned by the adapter, not by the client being retired. func TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow(t *testing.T) { featureKind := ldstoreimpl.Features() flagKey := st.Flag1ServerSide.Flag.Key @@ -523,7 +528,14 @@ func TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow(t *testing.T) { // Hypothesis 6: Behavior during the swap window (requests arriving mid-swap). // ----------------------------------------------------------------------------------------------- -func TestReanchorPoC_H6_AnchorPointerFlipsBeforeNewClientIsRegistered(t *testing.T) { +// TestReanchorPoC_H6_AnchorHoldsUntilNewClientReady is the inversion of the original H6 finding. +// The originally-observed broken behavior: reconcileCredentials flipped the rotator's anchor +// synchronously and then built the new client asynchronously, opening a window where GetClient() +// (== clients[AnchorKey()]) returned nil for the not-yet-registered new key. The fix builds the new +// client synchronously and only commits the anchor once it reports Initialized, so no such nil window +// exists: while the new client is still building, the anchor stays on the old key and GetClient() keeps +// returning the old, still-serving client. +func TestReanchorPoC_H6_AnchorHoldsUntilNewClientReady(t *testing.T) { envConfig := st.EnvMain.Config mockLog := ldlogtest.NewMockLog() @@ -533,10 +545,13 @@ func TestReanchorPoC_H6_AnchorPointerFlipsBeforeNewClientIsRegistered(t *testing inner := testclient.FakeLDClientFactoryWithChannel(true, clientCh) // gate blocks construction of the NEW anchor's client so we can observe the swap window - // deterministically (no sleeps / no racing). + // deterministically. entered signals that the synchronous build has reached the factory (and is now + // blocked on gate) — at which point the re-anchor is mid-flight but has not yet committed. gate := make(chan struct{}) + entered := make(chan struct{}, 1) gatedFactory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { if sdkKey == reanchorTestKey2 { + entered <- struct{}{} <-gate } return inner(sdkKey, cfg, timeout) @@ -550,30 +565,46 @@ func TestReanchorPoC_H6_AnchorPointerFlipsBeforeNewClientIsRegistered(t *testing client1 := requireClientReady(t, clientCh) require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) - // Re-anchor. reconcileCredentials flips the rotator's primary SDK key synchronously, then starts the - // new client on a background goroutine (which blocks in the factory on `gate`). + // The re-anchor is synchronous, so the reconcile blocks in the gated factory until we release it. + // Drive it on a background goroutine and build the set up front (keeping require off that goroutine). start := time.Unix(1000, 0) - reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) - - // FINDING: there is a window where the anchor pointer already names the new key but no client exists - // for it yet, so GetClient() returns nil. GetClient() == clients[rotator.AnchorKey()], and the rotator's - // anchor flipped to the new key before startSDKClient registered the client. A request arriving in - // this window gets a nil client. T2.c must not advance the anchor pointer until the new client is - // registered (and ideally Initialized()). - assert.Nil(t, env.GetClient(), "GetClient() is nil during the swap window") + set, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: reanchorTestKey2}). + WithSDKKey(credential.SDKKeyParams{Value: envConfig.SDKKey, Expiry: util.PtrOrNil(start.Add(time.Hour))}). + Build() + require.NoError(t, err) + done := make(chan struct{}) + go func() { + defer close(done) + env.(*envContextImpl).reconcileCredentials(set, start) + }() + + // The build has reached the factory and is blocked: the re-anchor is mid-flight, pre-commit. + <-entered + envImpl := env.(*envContextImpl) + assert.Same(t, client1, env.GetClient(), "GetClient() keeps returning the old client during the build — never nil") + assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "the anchor stays on the old key until the new client is ready") - // Release the gate; the new client registers and GetClient() recovers. + // Release the gate; the new client initializes, the anchor commits, and GetClient() advances. close(gate) + <-done client2 := requireClientReady(t, clientCh) require.Eventually(t, func() bool { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond, - "GetClient() recovers once the new client is registered") + "GetClient() returns the new client once it is registered and the anchor commits") + assert.Equal(t, reanchorTestKey2, envImpl.keyRotator.AnchorKey()) } // ----------------------------------------------------------------------------------------------- // Hypothesis 7: Failure modes — new client init fails. // ----------------------------------------------------------------------------------------------- -func TestReanchorPoC_H7_FailedNewClientLeavesEnvWithoutAnchorClient(t *testing.T) { +// TestReanchorPoC_H7_FailedNewClientRollsBackToOldAnchor is the inversion of the original H7 finding. +// The originally-observed broken behavior: a failed new-client init left the anchor already flipped to +// a key with no client, so GetClient() returned nil and the environment was broken even though the old +// client was still alive. The fix builds and validates the new client before committing: on init +// failure it does NOT commit the anchor, surfaces the error, and leaves the old anchor authoritative +// (its client keeps serving). This is the all-or-nothing atomicity requirement applied to re-anchor. +func TestReanchorPoC_H7_FailedNewClientRollsBackToOldAnchor(t *testing.T) { envConfig := st.EnvMain.Config fakeErr := errors.New("new anchor client failed to initialize") @@ -599,25 +630,25 @@ func TestReanchorPoC_H7_FailedNewClientLeavesEnvWithoutAnchorClient(t *testing.T client1 := requireClientReady(t, clientCh) require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) - // Re-anchor onto a key whose client init fails (old key kept valid for a grace hour). + // Re-anchor onto a key whose client init fails (old key kept valid for a grace hour). The re-anchor + // is synchronous, so the init failure and rollback are complete by the time reanchor returns. start := time.Unix(1000, 0) reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) - require.Eventually(t, func() bool { return env.GetInitError() != nil }, time.Second, 10*time.Millisecond, - "the failed new-client init should surface as an init error") - assert.Equal(t, fakeErr, env.GetInitError()) - - // FINDING: the rotator already flipped the anchor to the new key, but no client exists for it, so - // GetClient() returns nil -- even though the OLD anchor's client is still alive and valid during its - // grace period. A failed re-anchor breaks the environment with today's code. This is exactly the §8 - // atomicity requirement: T2.c must validate that the new client initializes BEFORE swapping the - // anchor pointer, and roll back to the old anchor on failure (preserving the previous accepted set). - assert.Nil(t, env.GetClient(), "GetClient() is nil after a failed re-anchor") - + // Post-fix: the re-anchor rolled back. The env stays healthy on the old anchor, so GetInitError + // stays nil — setting it would 401 a still-serving env at the request middleware. The failure + // surfaces via a structured Error log instead. The anchor pointer stayed on the old key, whose + // client is still alive and serving, so GetClient() never returns nil and no client is installed + // for the failed new anchor. + assert.NoError(t, env.GetInitError(), "a failed re-anchor must not mark the still-serving env as failed") + mockLog.AssertMessageMatch(t, true, ldlog.Error, "Re-anchor to SDK key .* failed") + assert.Same(t, client1, env.GetClient(), "GetClient() still returns the old anchor's client after rollback") envImpl := env.(*envContextImpl) + assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "the anchor pointer stays on the old key") envImpl.mu.RLock() _, oldStillPresent := envImpl.clients[envConfig.SDKKey] + _, newInstalled := envImpl.clients[reanchorTestKey2] envImpl.mu.RUnlock() - assert.True(t, oldStillPresent, - "the old anchor's client is still alive -- the data path could have been preserved by rolling back") + assert.True(t, oldStillPresent, "the old anchor's client is preserved") + assert.False(t, newInstalled, "no client is installed for the failed new anchor") } From a30e36fe749efa670d04e1c6e5a7180ab2966d6c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:01:23 -0400 Subject: [PATCH 35/66] fix(deps): bump supported Go versions to 1.26.5 and 1.25.12 (#740) --- .github/variables/go-versions.env | 4 ++-- Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/variables/go-versions.env b/.github/variables/go-versions.env index 954adc19..a90cd467 100644 --- a/.github/variables/go-versions.env +++ b/.github/variables/go-versions.env @@ -1,2 +1,2 @@ -latest=1.26.4 -penultimate=1.25.11 +latest=1.26.5 +penultimate=1.25.12 diff --git a/Dockerfile b/Dockerfile index f8ccbaac..29b9edf1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # This is a standalone Dockerfile that does not depend on goreleaser building the binary # It is NOT the version that is pushed to dockerhub -FROM golang:1.26.4-alpine3.24 as builder +FROM golang:1.26.5-alpine3.24 as builder # See "Runtime platform versions" in CONTRIBUTING.md RUN apk --no-cache add \ From 62a1fa8858a2bb4db131e2055f178ae212855494 Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Thu, 16 Jul 2026 12:47:46 -0400 Subject: [PATCH 36/66] feat: Re-anchor big-segment sync onto the new anchor (#741) Big-segment sync now follows SDK re-anchors instead of staying on the retired anchor key. On commitReanchor, reanchorBigSegmentSync rebuilds the synchronizer (key is fixed at construction), re-attaches an update consumer, starts the replacement immediately if big segments were already active, then closes the old instance. --- internal/relayenv/env_context_impl.go | 153 +++++--- internal/relayenv/env_context_impl_test.go | 9 +- .../env_context_reanchor_bigsegment_test.go | 328 ++++++++++++++++++ .../relayenv/env_context_reanchor_test.go | 34 +- 4 files changed, 459 insertions(+), 65 deletions(-) create mode 100644 internal/relayenv/env_context_reanchor_bigsegment_test.go diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 69f1fcff..9e017dd6 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -89,19 +89,23 @@ type EnvContextImplParams struct { } type envContextImpl struct { - mu sync.RWMutex - clients map[config.SDKKey]sdks.LDClientContext - storeAdapter *store.SSERelayDataStoreAdapter - loggers ldlog.Loggers - identifiers EnvIdentifiers - secureMode bool - envStreams *streams.EnvStreams - streamProviders []streams.StreamProvider - handlers map[streams.StreamProvider]map[credential.SDKCredential]http.Handler - jsContext JSClientContext - evaluator ldeval.Evaluator - eventDispatcher *events.EventDispatcher - bigSegmentSync bigsegments.BigSegmentSynchronizer + mu sync.RWMutex + clients map[config.SDKKey]sdks.LDClientContext + storeAdapter *store.SSERelayDataStoreAdapter + loggers ldlog.Loggers + identifiers EnvIdentifiers + secureMode bool + envStreams *streams.EnvStreams + streamProviders []streams.StreamProvider + handlers map[streams.StreamProvider]map[credential.SDKCredential]http.Handler + jsContext JSClientContext + evaluator ldeval.Evaluator + eventDispatcher *events.EventDispatcher + bigSegmentSync bigsegments.BigSegmentSynchronizer + // makeBigSegmentSync builds a BigSegmentSynchronizer for a given anchor SDK key, binding the + // construction-time inputs (http config, store, URIs, env ID, loggers). A re-anchor uses it to + // rebuild the synchronizer on the new anchor. nil when big segments are not configured. + makeBigSegmentSync func(anchor config.SDKKey) bigsegments.BigSegmentSynchronizer bigSegmentStore bigsegments.BigSegmentStore bigSegmentsExist bool sdkBigSegments *ldstoreimpl.BigSegmentStoreWrapper @@ -231,33 +235,20 @@ func NewEnvContext( if factory == nil { factory = bigsegments.DefaultBigSegmentSynchronizerFactory } - envContext.bigSegmentSync = factory( - httpConfig, bigSegmentStore, allConfig.Main.BaseURI.String(), allConfig.Main.StreamURI.String(), - envConfig.EnvID, envConfig.SDKKey, envLoggers, logPrefix) - thingsToCleanUp.AddFunc(envContext.bigSegmentSync.Close) - segmentUpdateCh := envContext.bigSegmentSync.SegmentUpdatesCh() - if segmentUpdateCh != nil { - go func() { - for range segmentUpdateCh { - // BigSegmentSynchronizer sends to this channel after processing a batch of - // big segment updates. The value it sends is a list of segment keys, but in - // the current implementation, we don't care what those keys are because we'll - // just be broadcasting a "ping" to all connected client-side SDKs. In the future - // if we have real evaluation streams, we'll need to determine which flags should - // be re-evaluated based on the segments. - if envContext.sdkBigSegments != nil { - envContext.sdkBigSegments.ClearCache() - } - if envContext.envStreams != nil { - envContext.envStreams.InvalidateClientSideState() - } - // If we shut down the environment, the BigSegmentSynchronizer will be closed which - // will also cause this channel to be closed, exiting this goroutine. - } - }() + // Bind the construction-time inputs so a re-anchor can rebuild the synchronizer on the new anchor + // key (see reanchorBigSegmentSync). The synchronizer authenticates from the SDK key it is handed + // (bigsegments/sync sets the Authorization header from it directly), so re-anchoring only needs + // the new key; httpConfig is transport configuration and is reused as-is. + baseURI := allConfig.Main.BaseURI.String() + streamURI := allConfig.Main.StreamURI.String() + envContext.makeBigSegmentSync = func(anchor config.SDKKey) bigsegments.BigSegmentSynchronizer { + return factory(httpConfig, bigSegmentStore, baseURI, streamURI, envConfig.EnvID, anchor, envLoggers, logPrefix) } - // We deliberate do not call bigSegmentSync.Start() here because we don't want the synchronizer to - // start until we know that at least one big segment exists. That's implemented by the + envContext.bigSegmentSync = envContext.makeBigSegmentSync(envConfig.SDKKey) + thingsToCleanUp.AddFunc(envContext.bigSegmentSync.Close) + envContext.consumeBigSegmentUpdates(envContext.bigSegmentSync) + // We deliberately do not call bigSegmentSync.Start() here because we don't want the synchronizer + // to start until we know that at least one big segment exists. That's implemented by the // envContextStreamUpdates methods. } @@ -796,11 +787,9 @@ func (c *envContextImpl) commitReanchor(newAnchor, previousAnchor config.SDKKey, c.eventDispatcher.ReplaceCredential(newAnchor) } - // Big-segment synchronization is intentionally left pointing at the previous anchor key across a - // re-anchor: this matches pre-concurrent-keys behavior (there was no re-anchor, so it never moved) - // and does not regress. When big-segment re-anchor is implemented, its re-wire hook belongs right - // here, after the event/metrics ReplaceCredential calls — either recreate the BigSegmentSynchronizer - // for newAnchor, or add a credential-replacement method to it. + // Re-wire big-segment synchronization onto the new anchor: its poll/stream requests authenticate + // with the anchor SDK key, so it must follow the anchor like the event/metrics forwarding above. + c.reanchorBigSegmentSync(newAnchor) c.globalLoggers.Infof("Re-anchored SDK from %s to %s (%s)", previousAnchor.Masked(), newAnchor.Masked(), why) return true @@ -1061,18 +1050,84 @@ func (c *envContextImpl) Close() error { return nil } +// consumeBigSegmentUpdates spawns a goroutine that drains sync's update channel, broadcasting a +// cache-clear + client-side invalidation for each batch. The goroutine exits when the channel closes +// (i.e. when sync is Closed). Called for the initial synchronizer and for each re-anchor replacement, so +// each synchronizer instance gets its own consumer bound to its own channel. +func (c *envContextImpl) consumeBigSegmentUpdates(sync bigsegments.BigSegmentSynchronizer) { + ch := sync.SegmentUpdatesCh() + if ch == nil { + return + } + go func() { + for range ch { + // The batch's segment keys are not needed today: we just ping all connected client-side SDKs. + // (A future evaluation-stream design would use the keys to target re-evaluation.) + if c.sdkBigSegments != nil { + c.sdkBigSegments.ClearCache() + } + if c.envStreams != nil { + c.envStreams.InvalidateClientSideState() + } + } + }() +} + +// reanchorBigSegmentSync rebuilds the big-segment synchronizer on the new anchor when the SDK anchor +// changes. The synchronizer bakes in its SDK key at construction and is not restartable, so re-anchoring +// recreates it rather than mutating it. The caller (commitReanchor) holds c.mu. +// +// If a big segment had already appeared (bigSegmentsExist -> the old synchronizer was Started), the +// replacement is Started immediately so synchronization continues without a gap. The old synchronizer is +// Closed last, which also ends its update-consumer goroutine. When big segments are not configured for +// this env there is no synchronizer and this is a no-op. sdkBigSegments (the SDK-facing store wrapper) +// persists across the re-anchor, so its polling-active state does not need re-setting here. +func (c *envContextImpl) reanchorBigSegmentSync(newAnchor config.SDKKey) { + if c.bigSegmentSync == nil { + return + } + wasStarted := c.bigSegmentsExist + old := c.bigSegmentSync + c.bigSegmentSync = c.makeBigSegmentSync(newAnchor) + c.consumeBigSegmentUpdates(c.bigSegmentSync) + if wasStarted { + c.bigSegmentSync.Start() + } + old.Close() +} + func (c *envContextImpl) setBigSegmentsExist() { c.mu.Lock() - alreadyExisted := c.bigSegmentsExist + firstTime := !c.bigSegmentsExist c.bigSegmentsExist = true + // Start the CURRENT synchronizer while holding the lock. Capturing the pointer and starting it after + // unlocking would let a concurrent re-anchor swap and Close that instance in between, so we'd Start() + // a synchronizer that was just retired. Starting c.bigSegmentSync under the lock guarantees we start + // whichever synchronizer is current -- the same one reanchorBigSegmentSync starts under this lock -- + // and never a retired one. Start() only launches a goroutine (it is non-blocking), so holding c.mu is + // fine, exactly as in reanchorBigSegmentSync. + started := firstTime && c.bigSegmentSync != nil + if started { + c.bigSegmentSync.Start() + } c.mu.Unlock() - if !alreadyExisted && c.bigSegmentSync != nil { - c.bigSegmentSync.Start() + if started { c.sdkBigSegments.SetPollingActive(true) // has no effect if already active } } +// bigSegmentSyncConfigured reports whether this env has a big-segment synchronizer. The field's +// nil-ness is invariant over the env's life (nil iff big segments were never configured; a re-anchor +// only swaps one non-nil synchronizer for another), but the read must still be synchronized against +// that concurrent reassign in reanchorBigSegmentSync -- the store-update sink below runs on the SDK +// data-source goroutine while a re-anchor runs on the reconcile goroutine. +func (c *envContextImpl) bigSegmentSyncConfigured() bool { + c.mu.RLock() + defer c.mu.RUnlock() + return c.bigSegmentSync != nil +} + func (q envContextStoreQueries) IsInitialized() bool { if s := q.context.storeAdapter.GetStore(); s != nil { return s.IsInitialized() @@ -1091,7 +1146,7 @@ func (u *envContextStreamUpdates) SendAllDataUpdate(allData []ldstoretypes.Colle // We use this delegator, rather than sending updates directory to context.envStreams, so that we // can detect the presence of a big segment and turn on the big segment synchronizer as needed. u.context.envStreams.SendAllDataUpdate(allData) - if u.context.bigSegmentSync == nil { + if !u.context.bigSegmentSyncConfigured() { return } @@ -1114,7 +1169,7 @@ func (u *envContextStreamUpdates) SendAllDataUpdate(allData []ldstoretypes.Colle func (u *envContextStreamUpdates) SendSingleItemUpdate(kind ldstoretypes.DataKind, key string, item ldstoretypes.ItemDescriptor) { // See comments in SendAllDataUpdate. u.context.envStreams.SendSingleItemUpdate(kind, key, item) - if u.context.bigSegmentSync == nil { + if !u.context.bigSegmentSyncConfigured() { return } hasBigSegment := false diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index ebd6a12a..b199ed0e 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -1194,8 +1194,15 @@ func (s *mockBigSegmentSynchronizer) SegmentUpdatesCh() <-chan bigsegments.Updat func (s *mockBigSegmentSynchronizer) Close() { s.lock.Lock() + defer s.lock.Unlock() + if s.closed { + return + } s.closed = true - s.lock.Unlock() + // Mirror the real synchronizer: Close closes the update channel, which is what terminates the + // consumer goroutine ranging over SegmentUpdatesCh(). Without this the tests would leak a consumer + // per re-anchor and never actually exercise the "old consumer exits on Close" invariant. + close(s.updateCh) } func (s *mockBigSegmentSynchronizer) isStarted() bool { diff --git a/internal/relayenv/env_context_reanchor_bigsegment_test.go b/internal/relayenv/env_context_reanchor_bigsegment_test.go new file mode 100644 index 00000000..5f06669a --- /dev/null +++ b/internal/relayenv/env_context_reanchor_bigsegment_test.go @@ -0,0 +1,328 @@ +package relayenv + +// Tests for T2.d (SDK-2543): the big-segment synchronizer follows the anchor across a re-anchor. +// TestReanchorPoC_H3_BigSegmentSyncFollowsAnchorOnReAnchor covers the basic "recreated on the new key, +// not yet started" case; these cover the started-continues, rollback, and not-configured cases. + +import ( + "net/http" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/basictypes" + "github.com/launchdarkly/ld-relay/v8/internal/bigsegments" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" + st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + "github.com/launchdarkly/ld-relay/v8/internal/streams" + + "github.com/launchdarkly/eventsource" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + ld "github.com/launchdarkly/go-server-sdk/v7" + "github.com/launchdarkly/go-server-sdk/v7/ldcomponents" + "github.com/launchdarkly/go-server-sdk/v7/subsystems" + helpers "github.com/launchdarkly/go-test-helpers/v3" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newBigSegmentTestEnv( + t *testing.T, + storeFactory bigsegments.BigSegmentStoreFactory, + clientFactory sdks.ClientFactoryFunc, + capturing *capturingBigSegmentSynchronizerFactory, + loggers ldlog.Loggers, +) EnvContext { + env, err := NewEnvContext(EnvContextImplParams{ + Identifiers: EnvIdentifiers{ConfiguredName: st.EnvMain.Name}, + EnvConfig: st.EnvMain.Config, + AllConfig: config.Config{}, + BigSegmentStoreFactory: storeFactory, + BigSegmentSynchronizerFactory: capturing.create, + ClientFactory: clientFactory, + SDKBigSegmentsConfigFactory: ldcomponents.BigSegments( + st.ExistingInstance[subsystems.BigSegmentStore](&st.NoOpSDKBigSegmentStore{}), + ), + ConnectionMapper: mockConnectionMapper{}, + Loggers: loggers, + }, nil) + require.NoError(t, err) + return env +} + +func nullBigSegmentStoreFactory(config.EnvConfig, config.Config, ldlog.Loggers) (bigsegments.BigSegmentStore, error) { + return bigsegments.NewNullBigSegmentStore(), nil +} + +// TestReanchorBigSegmentSync_StartedSyncContinuesAndOldClosed: once a big segment exists the +// synchronizer is Started; a re-anchor must recreate it on the new key, Start the replacement (so sync +// continues without a gap), and Close the old one. +func TestReanchorBigSegmentSync_StartedSyncContinuesAndOldClosed(t *testing.T) { + envConfig := st.EnvMain.Config + capturing := &capturingBigSegmentSynchronizerFactory{} + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + env := newBigSegmentTestEnv(t, nullBigSegmentStoreFactory, + testclient.FakeLDClientFactoryWithChannel(true, clientCh), capturing, mockLog.Loggers) + defer env.Close() + envImpl := env.(*envContextImpl) + + // A big segment appears -> the current synchronizer is Started. + envImpl.setBigSegmentsExist() + oldSync := capturing.latest() + require.True(t, oldSync.isStarted(), "the synchronizer is started once a big segment exists") + + // Re-anchor onto a new key (its client builds healthy, so the re-anchor commits). + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) + + count, sdkKey := capturing.snapshot() + assert.Equal(t, 2, count, "the synchronizer is recreated on re-anchor") + assert.Equal(t, reanchorTestKey2, sdkKey, "on the new anchor key") + newSync := capturing.latest() + assert.NotSame(t, oldSync, newSync, "a fresh synchronizer instance") + assert.True(t, newSync.isStarted(), "the replacement is Started so synchronization continues without a gap") + assert.True(t, oldSync.isClosed(), "the old synchronizer is Closed") + + // Closing the old synchronizer closes its update channel, which is what terminates its + // update-consumer goroutine (the `for range` in consumeBigSegmentUpdates). Verify the channel is + // closed so a re-anchor cannot leak a consumer per rotation. + _, ok := <-oldSync.updateCh + assert.False(t, ok, "the old synchronizer's update channel is closed, so its consumer goroutine exits") +} + +// TestReanchorBigSegmentSync_RollbackDoesNotRewire: if the re-anchor's new client fails to build, the +// re-anchor rolls back and the big-segment synchronizer must stay on the previous anchor (not recreated, +// not closed). +func TestReanchorBigSegmentSync_RollbackDoesNotRewire(t *testing.T) { + envConfig := st.EnvMain.Config + capturing := &capturingBigSegmentSynchronizerFactory{} + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + healthy := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + clientFactory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == reanchorTestKey2 { + // New anchor fails to init (non-nil uninitialized client + error, as the real SDK returns). + return &testclient.FakeLDClient{Key: sdkKey, CloseCh: make(chan struct{})}, ld.ErrInitializationFailed + } + return healthy(sdkKey, cfg, timeout) + } + + env := newBigSegmentTestEnv(t, nullBigSegmentStoreFactory, clientFactory, capturing, mockLog.Loggers) + defer env.Close() + envImpl := env.(*envContextImpl) + envImpl.setBigSegmentsExist() // synchronizer started + oldSync := capturing.latest() + + // Re-anchor to the failing key -> rollback. + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) + + count, sdkKey := capturing.snapshot() + assert.Equal(t, 1, count, "a rolled-back re-anchor must not recreate the synchronizer") + assert.Equal(t, envConfig.SDKKey, sdkKey, "it stays on the previous anchor key") + assert.Same(t, oldSync, capturing.latest(), "same synchronizer instance") + assert.False(t, oldSync.isClosed(), "the synchronizer is not closed by a rolled-back re-anchor") + assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "anchor unchanged after rollback") +} + +// TestReanchorBigSegmentSync_NotConfiguredIsNoOp: when big segments are not configured there is no +// synchronizer, and a re-anchor must be a no-op for big-segment sync (no creation, no panic). +func TestReanchorBigSegmentSync_NotConfiguredIsNoOp(t *testing.T) { + envConfig := st.EnvMain.Config + capturing := &capturingBigSegmentSynchronizerFactory{} + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + noStore := func(config.EnvConfig, config.Config, ldlog.Loggers) (bigsegments.BigSegmentStore, error) { + return nil, nil // no big-segment store -> no synchronizer + } + env := newBigSegmentTestEnv(t, noStore, + testclient.FakeLDClientFactoryWithChannel(true, clientCh), capturing, mockLog.Loggers) + defer env.Close() + + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) + + count, _ := capturing.snapshot() + assert.Equal(t, 0, count, "no synchronizer is created when big segments are not configured") + assert.Equal(t, reanchorTestKey2, env.(*envContextImpl).keyRotator.AnchorKey(), "the SDK re-anchor still committed") +} + +// TestReanchorBigSegmentSync_NewSyncDrivesClientSideInvalidation is the end-to-end integration case +// (SDK-2543 AC): with a client-side stream connected across a re-anchor, a big-segment update delivered +// on the NEW synchronizer must still ping the connected client -- proving the re-wired synchronizer's +// update consumer is active and drives client-side invalidation. +func TestReanchorBigSegmentSync_NewSyncDrivesClientSideInvalidation(t *testing.T) { + envConfig := st.EnvClientSide.Config + capturing := &capturingBigSegmentSynchronizerFactory{} + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + jsClientStreams := streams.NewStreamProvider(basictypes.JSClientPingStream, time.Hour, 0) + sdkStartedCh := make(chan EnvContext, 1) + clientCh := make(chan *testclient.FakeLDClient, 10) + env, err := NewEnvContext(EnvContextImplParams{ + Identifiers: EnvIdentifiers{ConfiguredName: st.EnvMain.Name}, + EnvConfig: envConfig, + AllConfig: config.Config{}, + BigSegmentStoreFactory: nullBigSegmentStoreFactory, + BigSegmentSynchronizerFactory: capturing.create, + ClientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientCh), + SDKBigSegmentsConfigFactory: ldcomponents.BigSegments( + st.ExistingInstance[subsystems.BigSegmentStore](&st.NoOpSDKBigSegmentStore{}), + ), + StreamProviders: []streams.StreamProvider{jsClientStreams}, + ConnectionMapper: mockConnectionMapper{}, + Loggers: mockLog.Loggers, + }, sdkStartedCh) + require.NoError(t, err) + defer env.Close() + + <-sdkStartedCh + _ = env.GetStore().Init(nil) // client-side endpoint only pings once the store is initialized + oldSync := capturing.latest() + + streamHandler := env.GetStreamHandler(jsClientStreams, envConfig.EnvID) + req, _ := http.NewRequest("GET", "", nil) + st.WithStreamRequest(t, req, streamHandler, func(eventCh <-chan eventsource.Event) { + initEvent := helpers.RequireValue(t, eventCh, time.Minute) + require.Equal(t, "ping", initEvent.Event()) + helpers.AssertNoMoreValues(t, eventCh, 100*time.Millisecond) + + // Re-anchor mid-subscription; the synchronizer is rebuilt on the new anchor. + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) + newSync := capturing.latest() + require.NotSame(t, oldSync, newSync, "the synchronizer was rebuilt on re-anchor") + + // A big-segment update on the NEW synchronizer pings the still-connected client-side stream. + newSync.updateCh <- bigsegments.UpdatesSummary{SegmentKeysUpdated: []string{"seg"}} + pingEvent := helpers.RequireValue(t, eventCh, time.Second) + assert.Equal(t, "ping", pingEvent.Event()) + }) +} + +// reanchorTestKey3 is a third anchor SDK key, used to drive A->B->C sequential re-anchors. +const reanchorTestKey3 = config.SDKKey("reanchor-poc-new-anchor-3") + +// TestReanchorBigSegmentSync_ReanchorBeforeFirstSegmentThenStartsNewSync covers the ordering where a +// re-anchor happens BEFORE any big segment has appeared (so the replacement is built but not started), +// and then the first segment appears. setBigSegmentsExist must start the CURRENT (new) synchronizer, +// never the retired one. +func TestReanchorBigSegmentSync_ReanchorBeforeFirstSegmentThenStartsNewSync(t *testing.T) { + envConfig := st.EnvMain.Config + capturing := &capturingBigSegmentSynchronizerFactory{} + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + env := newBigSegmentTestEnv(t, nullBigSegmentStoreFactory, + testclient.FakeLDClientFactoryWithChannel(true, clientCh), capturing, mockLog.Loggers) + defer env.Close() + envImpl := env.(*envContextImpl) + + oldSync := capturing.latest() + require.False(t, oldSync.isStarted(), "no big segment yet -> the synchronizer is not started") + + // Re-anchor before any segment appears: the replacement is built on the new key but NOT started. + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) + newSync := capturing.latest() + require.NotSame(t, oldSync, newSync, "a fresh synchronizer instance on the new key") + require.False(t, newSync.isStarted(), "still no segment -> the replacement is not started yet") + assert.True(t, oldSync.isClosed(), "the old synchronizer is Closed on re-anchor") + + // The first big segment now appears: the CURRENT (new) synchronizer must be started. + envImpl.setBigSegmentsExist() + assert.True(t, newSync.isStarted(), "the current synchronizer is started when the first segment appears") + assert.False(t, oldSync.isStarted(), "the retired synchronizer is never started") +} + +// TestReanchorBigSegmentSync_MultipleSequentialReanchors drives A->B->C and asserts each intermediate +// synchronizer is Closed, only the final one is current+Started, and the create bookkeeping tracks each +// anchor key. Guards against synchronizer/consumer accumulation and stale-key bugs across rotations. +func TestReanchorBigSegmentSync_MultipleSequentialReanchors(t *testing.T) { + envConfig := st.EnvMain.Config + capturing := &capturingBigSegmentSynchronizerFactory{} + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + env := newBigSegmentTestEnv(t, nullBigSegmentStoreFactory, + testclient.FakeLDClientFactoryWithChannel(true, clientCh), capturing, mockLog.Loggers) + defer env.Close() + envImpl := env.(*envContextImpl) + + envImpl.setBigSegmentsExist() + syncA := capturing.latest() + require.True(t, syncA.isStarted()) + + // A -> B. + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) + syncB := capturing.latest() + require.NotSame(t, syncA, syncB) + assert.True(t, syncA.isClosed(), "A is closed after A->B") + assert.True(t, syncB.isStarted(), "B is started (a segment already existed)") + + // B -> C. + reanchor(t, env, reanchorTestKey3, reanchorTestKey2, time.Unix(1000, 0)) + syncC := capturing.latest() + require.NotSame(t, syncB, syncC) + assert.True(t, syncB.isClosed(), "B is closed after B->C") + assert.True(t, syncC.isStarted(), "C is started") + + count, sdkKey := capturing.snapshot() + assert.Equal(t, 3, count, "one synchronizer per anchor: A, B, C") + assert.Equal(t, reanchorTestKey3, sdkKey, "the current synchronizer is on the final anchor key") + assert.Equal(t, reanchorTestKey3, envImpl.keyRotator.AnchorKey(), "the SDK anchor is C") +} + +// TestReanchorBigSegmentSync_ConcurrentStoreUpdateDuringReanchorIsRaceFree is a regression test for the +// data race introduced when re-anchor made c.bigSegmentSync runtime-mutable: the store-update sink reads +// that field to decide whether to check for big segments, on the SDK data-source goroutine, while a +// re-anchor reassigns it under c.mu on the reconcile goroutine. Without synchronizing the read, `go test +// -race` flags a data race. This test drives the real sink concurrently with a real re-anchor. +func TestReanchorBigSegmentSync_ConcurrentStoreUpdateDuringReanchorIsRaceFree(t *testing.T) { + envConfig := st.EnvMain.Config + capturing := &capturingBigSegmentSynchronizerFactory{} + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + env := newBigSegmentTestEnv(t, nullBigSegmentStoreFactory, + testclient.FakeLDClientFactoryWithChannel(true, clientCh), capturing, mockLog.Loggers) + defer env.Close() + envImpl := env.(*envContextImpl) + + // The sink wired into the store adapter; SendAllDataUpdate reads c.bigSegmentSync. + sink := &envContextStreamUpdates{context: envImpl} + + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + sink.SendAllDataUpdate(nil) // reads c.bigSegmentSync via bigSegmentSyncConfigured() + } + } + }() + + // Re-anchor concurrently: reanchorBigSegmentSync reassigns c.bigSegmentSync under c.mu. + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) + + close(stop) + <-done + + // Sanity: the re-anchor committed despite the concurrent sink traffic. + count, sdkKey := capturing.snapshot() + assert.Equal(t, 2, count, "the synchronizer was recreated on re-anchor") + assert.Equal(t, reanchorTestKey2, sdkKey, "on the new anchor key") +} diff --git a/internal/relayenv/env_context_reanchor_test.go b/internal/relayenv/env_context_reanchor_test.go index 22543955..c7f62af1 100644 --- a/internal/relayenv/env_context_reanchor_test.go +++ b/internal/relayenv/env_context_reanchor_test.go @@ -246,7 +246,11 @@ func TestReanchorPoC_H2_DownstreamConnectionSurvivesReAnchor(t *testing.T) { }, time.Second, 10*time.Millisecond, "rotation to the new anchor should be applied") // FINDING: the open client-side connection survives the swap and still delivers events. - synchronizer.updateCh <- bigsegments.UpdatesSummary{SegmentKeysUpdated: []string{"fake-segment-key"}} + // T2.d re-anchors the big-segment synchronizer, so a post-re-anchor update arrives on the CURRENT + // (rebuilt) synchronizer, not the retired one (whose channel is now closed). + current := fakeSynchronizerFactory.synchronizer + require.NotSame(t, synchronizer, current, "the synchronizer was rebuilt on re-anchor") + current.updateCh <- bigsegments.UpdatesSummary{SegmentKeysUpdated: []string{"fake-segment-key"}} pingEvent := helpers.RequireValue(t, eventCh, time.Second) assert.Equal(t, "ping", pingEvent.Event(), "downstream connection should survive the re-anchor") }) @@ -310,7 +314,14 @@ func (f *capturingBigSegmentSynchronizerFactory) snapshot() (int, config.SDKKey) return f.createCount, f.lastSDKKey } -func TestReanchorPoC_H3_BigSegmentSyncIsNotReWiredOnReAnchor(t *testing.T) { +// latest returns the most recently created synchronizer (the current one after a re-anchor rebuild). +func (f *capturingBigSegmentSynchronizerFactory) latest() *mockBigSegmentSynchronizer { + f.mu.Lock() + defer f.mu.Unlock() + return f.synchronizer +} + +func TestReanchorPoC_H3_BigSegmentSyncFollowsAnchorOnReAnchor(t *testing.T) { envConfig := st.EnvMain.Config fakeBigSegmentStoreFactory := func(config.EnvConfig, config.Config, ldlog.Loggers) (bigsegments.BigSegmentStore, error) { @@ -354,20 +365,13 @@ func TestReanchorPoC_H3_BigSegmentSyncIsNotReWiredOnReAnchor(t *testing.T) { return false }, time.Second, 10*time.Millisecond, "rotation to the new anchor should be applied") - // Give any (hypothetical) re-wire a chance to run. - require.Never(t, func() bool { - c, _ := capturing.snapshot() - return c != 1 - }, 200*time.Millisecond, 20*time.Millisecond, "synchronizer must not be recreated by the re-anchor") - - // FINDING: big-segment sync is wired to the SDK key at construction and is NOT re-wired by today's - // swap path -- the synchronizer is neither recreated nor told about the new key (the - // BigSegmentSynchronizer interface has no credential-replacement method). After re-anchor it keeps - // polling/streaming on the OLD anchor key. The big-segment re-wire (follow-up work) must add a - // re-wire path (a ReplaceCredential-style method) or recreate the synchronizer on each re-anchor. + // T2.d: the re-anchor recreates the big-segment synchronizer on the NEW anchor key, so its + // poll/stream requests authenticate with the current anchor instead of the retired one. The + // synchronizer bakes its SDK key in at construction and is not restartable, so re-anchoring rebuilds + // it. (reconcileCredentials -> commitReanchor -> reanchorBigSegmentSync runs synchronously.) count, sdkKey = capturing.snapshot() - assert.Equal(t, 1, count, "synchronizer was not recreated on re-anchor") - assert.Equal(t, envConfig.SDKKey, sdkKey, "synchronizer still references the old anchor key") + assert.Equal(t, 2, count, "the synchronizer is recreated on re-anchor") + assert.Equal(t, reanchorTestKey2, sdkKey, "the new synchronizer references the new anchor key") } // ----------------------------------------------------------------------------------------------- From 6239d6e7816abedca1a60e18cd908763bff2d02a Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Thu, 16 Jul 2026 13:24:23 -0400 Subject: [PATCH 37/66] feat: Build stream handlers on demand instead of per credential (#742) Stream handlers are no longer pre-built and cached per credential. GetStreamHandler now calls streamProvider.Handler with a scoped (filterKey, credential) on each request and returns that handler (or the existing 404 invalid-stream handler when the provider returns nil). The envContextImpl.handlers map and all logic that populated or pruned it at env init, credential add, and credential remove are gone. --- .../env_context_handler_fanout_test.go | 97 +++++++++++++++++++ internal/relayenv/env_context_impl.go | 45 +++------ internal/relayenv/env_context_impl_test.go | 4 +- 3 files changed, 112 insertions(+), 34 deletions(-) create mode 100644 internal/relayenv/env_context_handler_fanout_test.go diff --git a/internal/relayenv/env_context_handler_fanout_test.go b/internal/relayenv/env_context_handler_fanout_test.go new file mode 100644 index 00000000..6a12ba50 --- /dev/null +++ b/internal/relayenv/env_context_handler_fanout_test.go @@ -0,0 +1,97 @@ +package relayenv + +// Tests for T2.e (SDK-2544): stream handlers are no longer built or stored per credential. Instead +// GetStreamHandler resolves the request's credential to a scoped channel and asks the StreamProvider to +// build the handler on demand, scoping it with the env's (immutable) filter key. These tests exercise +// that on-demand path directly: that the provider is asked for the right scoped credential, that a valid +// credential yields the provider's handler, and that a credential the provider rejects (wrong kind) falls +// back to the 404 handler. End-to-end multi-key streaming through the full HTTP stack is covered by the +// relay-package concurrent-keys auth suite. + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" + "github.com/launchdarkly/ld-relay/v8/internal/streams" + + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeStreamProvider records the scoped credential passed to Handler and returns a caller-supplied +// handler for credentials it accepts (nil otherwise, mimicking a real provider rejecting the wrong +// credential kind). Register/Close are unused by GetStreamHandler. +type fakeStreamProvider struct { + handlerFn func(sdkauth.ScopedCredential) http.HandlerFunc + scopes []sdkauth.ScopedCredential +} + +func (f *fakeStreamProvider) Handler(scoped sdkauth.ScopedCredential) http.HandlerFunc { + f.scopes = append(f.scopes, scoped) + return f.handlerFn(scoped) +} + +func (f *fakeStreamProvider) Register(sdkauth.ScopedCredential, streams.EnvStoreQueries, ldlog.Loggers) streams.EnvStreamProvider { + return nil +} + +func (f *fakeStreamProvider) Close() {} + +func TestGetStreamHandler_BuildsOnDemandScopedWithEnvFilterKey(t *testing.T) { + const filter = config.FilterKey("my-filter") + + served := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(299) }) + sp := &fakeStreamProvider{ + handlerFn: func(scoped sdkauth.ScopedCredential) http.HandlerFunc { + // Accept SDK keys only, as a server-side provider would; reject other credential kinds. + if _, ok := scoped.SDKCredential.(config.SDKKey); ok { + return served + } + return nil + }, + } + + c := &envContextImpl{filterKey: filter} + + // A valid (right-kind) credential: the provider is asked for that credential scoped with the env's + // filter key, and its handler is returned as-is (no per-credential storage, built on the spot). + h := c.GetStreamHandler(sp, config.SDKKey("sdk-A")) + require.Len(t, sp.scopes, 1) + assert.Equal(t, filter, sp.scopes[0].FilterKey, "scoped with the env's filter key") + assert.Equal(t, config.SDKKey("sdk-A"), sp.scopes[0].SDKCredential, "scoped with the request credential") + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/", nil)) + assert.Equal(t, 299, rr.Code, "the provider's on-demand handler is returned for a valid credential") + + // A second, different credential resolves independently through the same code path -- there is no + // shared/cached per-credential handler, each call re-derives the scoped channel. + c.GetStreamHandler(sp, config.SDKKey("sdk-B")) + require.Len(t, sp.scopes, 2) + assert.Equal(t, config.SDKKey("sdk-B"), sp.scopes[1].SDKCredential) +} + +func TestGetStreamHandler_WrongKindCredentialServes404(t *testing.T) { + sp := &fakeStreamProvider{ + handlerFn: func(scoped sdkauth.ScopedCredential) http.HandlerFunc { + if _, ok := scoped.SDKCredential.(config.SDKKey); ok { + return http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}) + } + return nil // provider does not support this credential kind + }, + } + + c := &envContextImpl{filterKey: config.DefaultFilter} + + // A credential the provider rejects (returns nil for) must fall back to the invalid-stream 404 + // handler, exactly as the old per-credential map miss did. + h := c.GetStreamHandler(sp, config.MobileKey("mob-key")) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/", nil)) + assert.Equal(t, http.StatusNotFound, rr.Code) +} diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 9e017dd6..3bc2a042 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -97,7 +97,6 @@ type envContextImpl struct { secureMode bool envStreams *streams.EnvStreams streamProviders []streams.StreamProvider - handlers map[streams.StreamProvider]map[credential.SDKCredential]http.Handler jsContext JSClientContext evaluator ldeval.Evaluator eventDispatcher *events.EventDispatcher @@ -196,7 +195,6 @@ func NewEnvContext( loggers: envLoggers, secureMode: envConfig.SecureMode, streamProviders: params.StreamProviders, - handlers: make(map[streams.StreamProvider]map[credential.SDKCredential]http.Handler), jsContext: params.JSClientContext, sdkClientFactory: params.ClientFactory, sdkInitTimeout: allConfig.Main.InitTimeout.GetOrElse(config.DefaultInitTimeout), @@ -270,16 +268,6 @@ func NewEnvContext( for _, c := range allCreds { envStreams.AddCredential(c) } - for _, sp := range params.StreamProviders { - handlers := make(map[credential.SDKCredential]http.Handler) - for _, c := range allCreds { - h := sp.Handler(sdkauth.NewScoped(envContext.filterKey, c)) - if h != nil { - handlers[c] = h - } - } - envContext.handlers[sp] = handlers - } dataStoreFactory := params.DataStoreFactory if dataStoreFactory == nil { @@ -481,9 +469,6 @@ func (c *envContextImpl) removeCredential(oldCredential credential.SDKCredential defer c.mu.Unlock() c.connectionMapper.RemoveConnectionMapping(sdkauth.NewScoped(c.filterKey, oldCredential)) c.envStreams.RemoveCredential(oldCredential) - for _, handlers := range c.handlers { - delete(handlers, oldCredential) - } // See the comment in addCredential for more context. In offline mode, there's no need to close the SDK client // because our data comes from a file, not a streaming connection. if !c.offline { @@ -815,18 +800,13 @@ func (c *envContextImpl) rebuildEvaluator() { } // registerCredentialMappings wires relay's downstream-facing routing for cred: it registers the -// credential with the env's stream machinery, builds the per-stream-provider HTTP handlers, and adds -// the connection→env mapping, so incoming SDK/client connections that authenticate with cred are -// served by this env. It does NOT start the upstream SDK client or repoint event/metrics forwarding — -// those are anchor-only concerns owned by the callers (addCredential, and the re-anchor sequence). -// The caller must hold c.mu. +// credential with the env's stream machinery and adds the connection→env mapping, so incoming +// SDK/client connections that authenticate with cred are served by this env. Stream handlers are built +// on demand per request in GetStreamHandler, so there is nothing per-credential to construct here. It +// does NOT start the upstream SDK client or repoint event/metrics forwarding — those are anchor-only +// concerns owned by the callers (addCredential, and the re-anchor sequence). The caller must hold c.mu. func (c *envContextImpl) registerCredentialMappings(cred credential.SDKCredential) { c.envStreams.AddCredential(cred) - for streamProvider, handlers := range c.handlers { - if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, cred)); h != nil { - handlers[cred] = h - } - } c.connectionMapper.AddConnectionMapping(sdkauth.NewScoped(c.filterKey, cred), c) } @@ -919,13 +899,14 @@ func (c *envContextImpl) GetLoggers() ldlog.Loggers { } func (c *envContextImpl) GetStreamHandler(streamProvider streams.StreamProvider, credential credential.SDKCredential) http.Handler { - c.mu.RLock() - defer c.mu.RUnlock() - h := c.handlers[streamProvider][credential] - if h == nil { - return http.HandlerFunc(invalidStreamHandler) - } - return h + // Build the handler on demand rather than storing one per (credential, provider): every handler in a + // (filter, provider) slot is identical except for the credential-derived channel id, which we resolve + // here from the request's already-authenticated credential. c.filterKey is immutable after + // construction, so this needs no lock. + if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, credential)); h != nil { + return h + } + return http.HandlerFunc(invalidStreamHandler) } func invalidStreamHandler(w http.ResponseWriter, req *http.Request) { diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index b199ed0e..a08dbdda 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -432,8 +432,8 @@ func TestNonAnchorSDKKeysDoNotOpenUpstreamClient(t *testing.T) { nonAnchorKey2 := config.SDKKey("non-anchor-key-2") // Reconcile to anchor + 2 non-anchor SDK keys. The anchor is unchanged, so no new anchor client - // is needed. Non-anchor keys must get envStreams + handlers + connection mapping but must NOT - // open an upstream client. + // is needed. Non-anchor keys must get envStreams + connection mapping but must NOT open an upstream + // client. env.ReconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). From bbc15992ce523e4b3c1db7945569f7e4be4ae221 Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Thu, 16 Jul 2026 13:42:36 -0400 Subject: [PATCH 38/66] fix: Close the demoted anchor's SDK client when a re-anchor commits (#744) --- internal/relayenv/env_context_impl.go | 38 ++++- internal/relayenv/env_context_impl_test.go | 72 ++++----- .../env_context_reanchor_synchronous_test.go | 81 ++++++---- .../relayenv/env_context_reanchor_test.go | 13 +- relay/autoconfig_key_change_test.go | 19 ++- ...autoconfig_key_rotation_end_to_end_test.go | 138 ++++++++++++++++++ 6 files changed, 267 insertions(+), 94 deletions(-) create mode 100644 relay/autoconfig_key_rotation_end_to_end_test.go diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 3bc2a042..39d4971c 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -581,9 +581,10 @@ func (c *envContextImpl) ReconcileCredentials(newSet credential.AcceptedSet) { // reference time for expiry math). // // Order: add -> re-anchor -> remove. Adding first registers the new keys' mappings; the re-anchor then -// swaps the upstream client while the old anchor is still serving; removing last tears down the old -// anchor (and any revoked keys) only once the new one is up. addCredential opens an upstream client -// only for the anchor -- non-anchor server keys are routed without a second connection. +// swaps the upstream client while the old anchor is still serving, closing the old anchor's client +// once the new one is committed; removing last tears down revoked keys' mappings only once the new +// anchor is up. addCredential opens an upstream client only for the anchor -- non-anchor server keys +// are routed without a second connection. // // reconcileMu serializes this whole method against concurrent reconciles and the cleanup ticker (see // triggerCredentialChanges). See reanchor for the SDK-anchor swap; MobilePrimaryRepoint is handled @@ -648,12 +649,17 @@ func (c *envContextImpl) reconcileCredentials(newSet credential.AcceptedSet, now // mappings if the key is brand new (Reconcile stripped it from additions), build a new SDK client, // and on Initialized commit the anchor. On init failure, roll back: do not commit, leave the previous // anchor authoritative (its client keeps serving), and log a structured error. -// - When a client already exists (e.g. a former anchor still in its grace period), or the env is -// offline: no build, just commit. +// - When a client already exists for the new anchor, or the env is offline: no build, just commit. +// (A demoted former anchor no longer has a client -- it was closed when its demotion committed -- +// so re-promoting an in-grace key builds a fresh client.) // // Returns true if the anchor was committed, false if it rolled back (init failure or the env closed -// mid-build), so reconcileCredentials can back out the anchor change. The old anchor's client is not -// closed here; its grace-period expiration drives removeCredential. +// mid-build), so reconcileCredentials can back out the anchor change. On commit, the previous +// anchor's client is closed here: the anchor owns the environment's single upstream connection, and +// leaving the demoted key's client running would hold a second upstream stream feeding the same +// shared store wrapper, broadcasting every update twice to connected clients. Only the client goes; +// the demoted key's credential mappings stay registered so it keeps authenticating downstream +// connections until its grace period expires (removeCredential then finds no client to close). func (c *envContextImpl) reanchor(change *credential.AnchorChange) bool { newAnchor := change.NewAnchor previousAnchor := change.PreviousAnchor @@ -711,7 +717,23 @@ func (c *envContextImpl) reanchor(change *credential.AnchorChange) bool { } } - return c.commitReanchor(newAnchor, previousAnchor, why) + if !c.commitReanchor(newAnchor, previousAnchor, why) { + return false + } + + // The new anchor's client is now authoritative, so tear down the previous anchor's client + // whether the key was grace-demoted or revoked outright (an undefined previous anchor has no + // entry, and a rolled-back commit never reaches here). The shared store wrapper survives: it is + // refcounted and the new anchor's client holds it. Offline mode is exempt, mirroring + // removeCredential: the offline branch above built no replacement, and the env's single + // file-data client (found by GetClient's map iteration) must keep serving across rotations. + if !c.offline { + if oldClient := c.clients[previousAnchor]; oldClient != nil { + delete(c.clients, previousAnchor) + _ = oldClient.Close() + } + } + return true } // buildNewAnchorClient constructs the SDK client for a re-anchor to newAnchor. It must run without c.mu diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index a08dbdda..cf7eab5b 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -342,27 +342,23 @@ func TestChangeSDKKey(t *testing.T) { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond, "env.GetClient() should return client2 after rotation") - // The client for the original SDK key should not have been closed, since it's valid for an hour. - if !helpers.AssertChannelNotClosed(t, client1.CloseCh, 1*time.Second, "client for envConfig.SDKKey should not have been closed yet") { + // The demoted key's client is closed as soon as the rotation commits: the anchor owns the env's + // single upstream connection, and leaving the old client running would broadcast every update + // twice through the shared store wrapper. The key itself stays accepted (asserted above), so + // downstream clients can still authenticate with it during the grace window. + if !helpers.AssertChannelClosed(t, client1.CloseCh, 1*time.Second, "client for envConfig.SDKKey should have been closed at rotation") { t.FailNow() } - // Simulate an amount of time passing that is less than the expiry window. The original key should still be valid. + // Simulate an amount of time passing that is less than the expiry window. The original key is + // still accepted for authentication. envImpl.triggerCredentialChanges(start.Add(45 * time.Minute)) - if !helpers.AssertChannelNotClosed(t, client1.CloseCh, 1*time.Second, "client for envConfig.SDKKey should not have been closed yet") { - t.FailNow() - } + assert.Contains(t, env.GetCredentials(), credential.SDKCredential(envConfig.SDKKey)) - // We are now an instant after the expiry. This should cause the original key to be removed - // and trigger its client to close. + // We are now an instant after the expiry. This should cause the original key to be removed. envImpl.triggerCredentialChanges(start.Add(1*time.Hour + 1*time.Millisecond)) assert.Equal(t, []credential.SDKCredential{key2}, env.GetCredentials()) assert.Empty(t, env.GetDeprecatedCredentials()) - - if !helpers.AssertChannelClosed(t, client1.CloseCh, 1*time.Second, "client for envConfig.SDKKey should have been closed") { - t.FailNow() - } - } // TestMobileKeyReconcileExpiry drives a mobile key carrying a per-key expiry end-to-end through the @@ -551,15 +547,14 @@ func TestNonPrimaryMobileKeyDoesNotStealEventForwarding(t *testing.T) { }) } -// When an SDK key that is still alive in its grace period is re-anchored back into the primary slot, -// a fresh SDK client is started for it. The previously-created client for that same key must be closed -// rather than silently dropped from the clients map, otherwise its upstream connection leaks. -// Originally a regression test from #716 for the old UpdateCredential path, where re-anchoring to a -// key still in its grace period spawned a fresh client and orphaned the old one. Under the -// ReconcileCredentials model that leak is structurally impossible: re-anchoring to a still-accepted key -// emits no "addition", so its existing client is reused rather than re-spawned, and the displaced -// anchor's client is closed by removeCredential. This test now verifies that reuse-and-no-leak guarantee. -func TestReAnchoringToKeyStillInGraceReusesItsClient(t *testing.T) { +// When an SDK key that is still accepted in its grace period is re-anchored back into the primary +// slot, a fresh SDK client is built for it (its previous client was closed when its demotion +// committed -- the anchor owns the env's single upstream connection, so a demoted key keeps only its +// credential mappings). Originally a regression test from #716 for the old UpdateCredential path, +// where re-anchoring to a key still in its grace period spawned a fresh client and orphaned the old +// one. Under the ReconcileCredentials model that leak is structurally impossible: every displaced +// anchor's client is closed at commit, so no rotation sequence can leave two live upstream clients. +func TestReAnchoringToKeyStillInGraceBuildsFreshClient(t *testing.T) { envConfig := st.EnvMain.Config keyA := envConfig.SDKKey keyB := config.SDKKey("keyB") @@ -580,8 +575,8 @@ func TestReAnchoringToKeyStillInGraceReusesItsClient(t *testing.T) { start := time.Unix(1000, 0) - // Rotate keyA -> keyB, deprecating keyA with an hour-long grace. keyA's client (clientA1) stays - // alive because keyA is still accepted during the grace window. + // Rotate keyA -> keyB, deprecating keyA with an hour-long grace. keyA stays accepted during the + // grace window, but its client (clientA1) is closed as soon as the re-anchor commits. env.(*envContextImpl).reconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). WithAnchor(credential.SDKKeyParams{Value: keyB}). @@ -590,35 +585,28 @@ func TestReAnchoringToKeyStillInGraceReusesItsClient(t *testing.T) { clientB := requireClientReady(t, clientCh) assert.NotEqual(t, clientA1, clientB) - if !helpers.AssertChannelNotClosed(t, clientA1.CloseCh, time.Second, "clientA1 should still be alive during keyA's grace") { + if !helpers.AssertChannelClosed(t, clientA1.CloseCh, time.Second, "clientA1 should have been closed when keyA was demoted") { t.FailNow() } - // Re-anchor back to keyA while it is still within its grace period. Because keyA is still an accepted - // credential, its existing client (clientA1) is reused as the anchor client rather than a new one - // being started -- so there is no stale client to orphan. keyB is omitted from the set (no expiry), - // so it is revoked immediately and its client is closed. + // Re-anchor back to keyA while it is still within its grace period. keyA's credential mappings + // survived the demotion but its client did not, so a fresh client is built for it. keyB is omitted + // from the set (no expiry), so it is revoked immediately; its client closes at commit. env.(*envContextImpl).reconcileCredentials( mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(credential.SDKKeyParams{Value: keyA})), start.Add(10*time.Minute)) - // keyB was revoked by the re-anchor, so its client is closed. - if !helpers.AssertChannelClosed(t, clientB.CloseCh, time.Second, "client for the revoked keyB should have been closed") { - t.FailNow() - } - // clientA1 is reused, not closed or churned: re-anchoring to a still-accepted key must not tear down - // its working upstream connection. - if !helpers.AssertChannelNotClosed(t, clientA1.CloseCh, time.Second, "clientA1 should be reused as the anchor client, not closed") { - t.FailNow() - } - // No new client is started for keyA -- the existing one is reused. - if !helpers.AssertNoMoreValues(t, clientCh, time.Second, "re-anchoring to an in-grace key must not start a new client") { + // A fresh client was built for keyA -- the demotion closed its original one. + clientA2 := requireClientReady(t, clientCh) + assert.NotEqual(t, clientA1, clientA2) + // keyB was displaced by the re-anchor, so its client is closed. + if !helpers.AssertChannelClosed(t, clientB.CloseCh, time.Second, "client for the displaced keyB should have been closed") { t.FailNow() } require.Eventually(t, func() bool { - return env.GetClient() == clientA1 - }, time.Second, 10*time.Millisecond, "env.GetClient() should return the reused client for keyA after re-anchor") + return env.GetClient() == clientA2 + }, time.Second, 10*time.Millisecond, "env.GetClient() should return the fresh client for keyA after re-anchor") creds := env.GetCredentials() assert.Contains(t, creds, keyA) diff --git a/internal/relayenv/env_context_reanchor_synchronous_test.go b/internal/relayenv/env_context_reanchor_synchronous_test.go index 9c0ca5fc..dee1a0bf 100644 --- a/internal/relayenv/env_context_reanchor_synchronous_test.go +++ b/internal/relayenv/env_context_reanchor_synchronous_test.go @@ -2,10 +2,11 @@ package relayenv // Regression tests for the synchronous re-anchor sequence. // -// These cover: re-anchoring to a new key (build success and init-failure rollback), re-anchoring to a -// previously-accepted key (reuse the existing client), no orphan clients, store-handover survival, and -// the mobile-primary repoint signal (the gap when the new primary mobile key was already in the -// accepted set, so the primary-mobile gate does not fire for it). +// These cover: re-anchoring to a new key (build success and init-failure rollback), re-anchoring back +// to a previously-accepted key (fresh client build -- the demoted key's client was closed when its +// demotion committed), no orphan clients, store-handover survival, and the mobile-primary repoint +// signal (the gap when the new primary mobile key was already in the accepted set, so the +// primary-mobile gate does not fire for it). import ( "errors" @@ -22,6 +23,7 @@ import ( "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" ld "github.com/launchdarkly/go-server-sdk/v7" "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" + helpers "github.com/launchdarkly/go-test-helpers/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -107,13 +109,18 @@ func TestReanchorSync_CaseA_BuildsNewClientAndMovesAnchor(t *testing.T) { require.NoError(t, err) assert.NotNil(t, got.Item, "data survives the re-anchor (no empty-store window)") - // The old client is still alive — its grace period has not elapsed (the old client keeps serving; - // closure happens via removeCredential when the expiry fires). + // The demoted old anchor's client is closed as part of the commit: the anchor owns the env's + // single upstream connection, and a second live stream would feed the shared store wrapper and + // broadcast every update twice to connected clients. Only the client goes -- the key itself stays + // accepted, so it keeps authenticating downstream connections during its grace period. + originalClient.AwaitClose(t, time.Second) envImpl := env.(*envContextImpl) envImpl.mu.RLock() _, oldStillPresent := envImpl.clients[envConfig.SDKKey] envImpl.mu.RUnlock() - assert.True(t, oldStillPresent, "old anchor's client retained during its grace period") + assert.False(t, oldStillPresent, "demoted old anchor's client is removed at commit") + assert.Contains(t, env.GetCredentials(), credential.SDKCredential(envConfig.SDKKey), + "the demoted key remains accepted for downstream auth during its grace period") } // TestReanchorSync_CaseA_InitFailureRollsBack confirms that a failed new-client init does NOT move @@ -167,11 +174,12 @@ func TestReanchorSync_CaseA_InitFailureRollsBack(t *testing.T) { assert.True(t, oldStillPresent, "old anchor's client preserved on rollback") } -// TestReanchorSync_CaseB_ReusesExistingClient covers re-anchoring onto a key that already has a -// live client. The simplest deterministic setup: re-anchor A→B (B's client built), then re-anchor -// B→A while A is still in its grace period. A's client still exists, so the second re-anchor must -// reuse it and build nothing new. -func TestReanchorSync_CaseB_ReusesExistingClient(t *testing.T) { +// TestReanchorSync_CaseB_RepromoteInGraceKeyBuildsFreshClient covers re-anchoring back onto a +// previously-accepted key: re-anchor A→B (B's client built, A's client closed at commit), then +// re-anchor B→A while A is still in its grace period. A's credential mappings survived the +// demotion, but its client did not, so the second re-anchor must build a fresh client for A and +// close B's client at commit. +func TestReanchorSync_CaseB_RepromoteInGraceKeyBuildsFreshClient(t *testing.T) { envConfig := st.EnvMain.Config mockLog := ldlogtest.NewMockLog() @@ -193,26 +201,23 @@ func TestReanchorSync_CaseB_ReusesExistingClient(t *testing.T) { key2Client := requireClientReady(t, clientCh) require.Same(t, key2Client, env.GetClient()) - // The original anchor's client is still alive in its grace period. + // The original anchor's client was closed at commit; only its credential mappings survive. + originalClient.AwaitClose(t, time.Second) envImpl := env.(*envContextImpl) envImpl.mu.RLock() - originalStillPresent := envImpl.clients[envConfig.SDKKey] == originalClient + _, originalStillPresent := envImpl.clients[envConfig.SDKKey] envImpl.mu.RUnlock() - require.True(t, originalStillPresent, "original client retained for reuse") + require.False(t, originalStillPresent, "demoted original anchor's client removed at commit") - // Second re-anchor: key2 → original. The original's client exists, so this is the reuse path: no - // Build, the existing client is reused, the anchor flips, and ReplaceCredential runs. + // Second re-anchor: key2 → original. The original key is still accepted (its mappings were never + // torn down) but it has no client, so a fresh one is built; key2's client closes at commit. reanchorViaReconcile(t, env, envConfig.SDKKey, reanchorSyncTestKey2, "", envConfig.MobileKey, envConfig.EnvID, now) - // No new client was created — clientCh must be empty (every prior client was drained). - select { - case c := <-clientCh: - t.Fatalf("re-anchoring to a key with an existing client must not build a new one, but one was created: %v", c.Key) - case <-time.After(100 * time.Millisecond): - } - - assert.Same(t, originalClient, env.GetClient(), "reuses the existing client for the re-anchored key") + freshClient := requireClientReady(t, clientCh) + assert.NotSame(t, originalClient, freshClient, "re-promoting an in-grace key builds a fresh client") + assert.Same(t, freshClient, env.GetClient(), "the fresh client is current after the re-anchor") assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "anchor flipped back to the original key") + key2Client.AwaitClose(t, time.Second) } // TestReanchorSync_CredentialExpiryDuringReanchorIsSerialized exercises the concurrency gap closed @@ -224,7 +229,8 @@ func TestReanchorSync_CaseB_ReusesExistingClient(t *testing.T) { // The test wedges a re-anchor open by blocking the new anchor's client build, then fires the ticker // from another goroutine and asserts (a) the ticker is blocked while the re-anchor holds reconcileMu, // (b) it completes once the re-anchor releases it, and (c) the final state is consistent — the new -// anchor committed, the expiring non-anchor key dropped, the demoted old anchor retained in grace. +// anchor committed, the expiring non-anchor key dropped, the demoted old anchor still accepted in +// grace (though its client was closed when the re-anchor committed). func TestReanchorSync_CredentialExpiryDuringReanchorIsSerialized(t *testing.T) { envConfig := st.EnvMain.Config @@ -233,7 +239,7 @@ func TestReanchorSync_CredentialExpiryDuringReanchorIsSerialized(t *testing.T) { now := time.Unix(2000, 0) expiringExpiry := now.Add(30 * time.Minute) // the non-anchor key the ticker will drop - graceExpiry := now.Add(2 * time.Hour) // the demoted old anchor stays alive in its grace period + graceExpiry := now.Add(2 * time.Hour) // the demoted old anchor stays accepted in its grace period tickerTime := now.Add(time.Hour) // between the two expiries: drops only the expiring key // The new anchor's client build blocks until releaseBuild is closed, holding the re-anchor (and @@ -327,10 +333,13 @@ func TestReanchorSync_CredentialExpiryDuringReanchorIsSerialized(t *testing.T) { assert.NotContains(t, env.GetCredentials(), credential.SDKCredential(reanchorSyncExpiringKey), "the expiring non-anchor key was dropped by the ticker") + originalClient.AwaitClose(t, time.Second) envImpl.mu.RLock() _, oldStillPresent := envImpl.clients[envConfig.SDKKey] envImpl.mu.RUnlock() - assert.True(t, oldStillPresent, "demoted old anchor's client retained during its grace period") + assert.False(t, oldStillPresent, "demoted old anchor's client closed when the re-anchor committed") + assert.Contains(t, env.GetCredentials(), credential.SDKCredential(envConfig.SDKKey), + "demoted old anchor remains accepted for downstream auth during its grace period") } // TestReanchorSync_MobilePrimaryRepoint_AlreadyAcceptedKey covers the primary-mobile gate's gap: @@ -450,8 +459,10 @@ func TestReanchorSync_PreviouslyAcceptedNonAnchorPromotedToAnchor(t *testing.T) // TestReanchorSync_Offline_CommitsWithoutBuildingClient covers the offline re-anchor branch: when the // env is offline, re-anchoring to a new key must commit the anchor WITHOUT building a new upstream -// client. (The initial anchor client is still created at startup; offline only skips the re-anchor -// build.) +// client, and the environment's single file-data client must keep serving. (The initial anchor +// client is still created at startup; offline skips both the re-anchor build and the demoted-anchor +// client teardown -- closing the only client with no replacement would flip /status to disconnected +// and, with a persistent store, tear down the backing store.) func TestReanchorSync_Offline_CommitsWithoutBuildingClient(t *testing.T) { envConfig := st.EnvMain.Config envConfig.Offline = true @@ -465,7 +476,8 @@ func TestReanchorSync_Offline_CommitsWithoutBuildingClient(t *testing.T) { defer env.Close() require.Equal(t, env, requireEnvReady(t, readyCh)) - _ = requireClientReady(t, clientCh) // drain the initial anchor client + initialClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == initialClient }, time.Second, 10*time.Millisecond) now := time.Unix(2000, 0) reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) @@ -478,6 +490,13 @@ func TestReanchorSync_Offline_CommitsWithoutBuildingClient(t *testing.T) { case <-time.After(100 * time.Millisecond): } assert.NoError(t, env.GetInitError()) + + // The single offline client survives the rotation: it is not closed and GetClient still finds it. + if !helpers.AssertChannelNotClosed(t, initialClient.CloseCh, 100*time.Millisecond, + "the offline env's only client must not be closed by a re-anchor") { + t.FailNow() + } + assert.Same(t, initialClient, env.GetClient(), "GetClient keeps returning the offline client after re-anchor") } // TestReanchorSync_RollbackWithImmediateRevocationKeepsOldAnchorServing covers the edge where a diff --git a/internal/relayenv/env_context_reanchor_test.go b/internal/relayenv/env_context_reanchor_test.go index c7f62af1..e061952f 100644 --- a/internal/relayenv/env_context_reanchor_test.go +++ b/internal/relayenv/env_context_reanchor_test.go @@ -94,10 +94,11 @@ func (f *sharedStoreFactory) Build(_ subsystems.ClientContext) (subsystems.DataS return f.store, nil } -// reanchor re-anchors env onto newKey while keeping oldKey valid for a grace hour (so the old client -// is not torn down during the swap). This mirrors the backend's default-rotation behavior: the new -// anchor is non-expiring, the demoted old anchor carries an expiry. It drives the time-injectable -// reconcileCredentials directly so the grace-period math is deterministic. +// reanchor re-anchors env onto newKey while keeping oldKey accepted for a grace hour (the old +// client stays up while the new one is built, then closes when the commit lands). This mirrors the +// backend's default-rotation behavior: the new anchor is non-expiring, the demoted old anchor +// carries an expiry. It drives the time-injectable reconcileCredentials directly so the +// grace-period math is deterministic. func reanchor(t *testing.T, env EnvContext, newKey, oldKey config.SDKKey, now time.Time) { t.Helper() set, err := credential.NewAcceptedSetBuilder(). @@ -447,8 +448,8 @@ func TestReanchorPoC_H5_StoreSurvivesReAnchor(t *testing.T) { require.NoError(t, err) require.NotNil(t, got.Item) - // Re-anchor onto a new key (old key kept valid for a grace hour, so the old client is not closed -- - // i.e. this exercises the recommended "start-new-before-close-old" ordering). + // Re-anchor onto a new key (old key kept accepted for a grace hour; the old client serves while + // the new one is built and closes once the commit lands -- the "start-new-before-close-old" ordering). start := time.Unix(1000, 0) reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) diff --git a/relay/autoconfig_key_change_test.go b/relay/autoconfig_key_change_test.go index 0e95dd39..0d0b0fbd 100644 --- a/relay/autoconfig_key_change_test.go +++ b/relay/autoconfig_key_change_test.go @@ -126,9 +126,10 @@ func TestAutoConfigUpdateEnvironmentSDKKeyWithExpiry(t *testing.T) { p.assertEnvLookup(env, testAutoConfEnv1.params()) // looking up env by old key still works assert.Equal(t, []credential.SDKCredential{testAutoConfEnv1.sdkKey.Value}, env.GetDeprecatedCredentials()) - if !helpers.AssertChannelNotClosed(t, client1.CloseCh, time.Millisecond*300, "should not have closed client for deprecated key yet") { - t.FailNow() - } + // The deprecated key stays valid for downstream auth (asserted above), but its upstream client + // closes as soon as the rotation commits: the new anchor owns the env's single upstream + // connection, and a second live stream would broadcast every update twice. + client1.AwaitClose(t, time.Second) }) } @@ -203,13 +204,17 @@ func TestAutoConfigRemovesCredentialForExpiredSDKKey(t *testing.T) { foundEnvWithOldKey, _ := p.relay.getEnvironment(sdkauth.New(oldKey)) assert.Equal(t, env, foundEnvWithOldKey) - if !helpers.AssertChannelClosed(t, client1.CloseCh, time.Duration(briefExpiryMillis+100)*time.Millisecond, "timed out waiting for client with old key to close") { - t.FailNow() - } + // The old key's client closes when the rotation commits, well before the expiry fires, so the + // client close is no longer a signal of the expiry itself. + client1.AwaitClose(t, time.Second) // After expiry, old key is removed; new key + mobile key + env ID are the only credentials left. + // The cleanup ticker drives the removal, so poll until it lands. expectedAfterExpiry := credentialsAsSet(modified.id, modified.mobKey, modified.SDKKey()) - assert.Equal(t, expectedAfterExpiry, credentialsAsSet(env.GetCredentials()...)) + require.Eventually(t, func() bool { + return assert.ObjectsAreEqual(expectedAfterExpiry, credentialsAsSet(env.GetCredentials()...)) + }, time.Duration(briefExpiryMillis)*time.Millisecond+time.Second, 20*time.Millisecond, + "timed out waiting for the expired key to be removed") noEnv, _ := p.relay.getEnvironment(sdkauth.New(oldKey)) assert.Nil(t, noEnv) }) diff --git a/relay/autoconfig_key_rotation_end_to_end_test.go b/relay/autoconfig_key_rotation_end_to_end_test.go new file mode 100644 index 00000000..18f19731 --- /dev/null +++ b/relay/autoconfig_key_rotation_end_to_end_test.go @@ -0,0 +1,138 @@ +package relay + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + + "github.com/launchdarkly/eventsource" + "github.com/launchdarkly/go-configtypes" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + "github.com/launchdarkly/go-sdk-common/v3/ldtime" + "github.com/launchdarkly/go-server-sdk-evaluation/v3/ldbuilders" + "github.com/launchdarkly/go-server-sdk/v7/testhelpers/ldservices" + helpers "github.com/launchdarkly/go-test-helpers/v3" + "github.com/launchdarkly/go-test-helpers/v3/httphelpers" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestAutoConfigKeyRotationClosesOldClientAndDoesNotDuplicateUpdates is an end-to-end regression +// test for the double-broadcast bug: after an auto-config key rotation demoted the old anchor with +// a grace-period expiry, the old anchor's SDK client kept its upstream stream open alongside the +// new anchor's client. Both clients fed the same shared store wrapper, so every upstream flag +// update was broadcast twice to connected downstream clients until the old key finally expired. +// +// Unlike the other auto-config tests, this one uses REAL SDK clients against a fake LaunchDarkly +// streaming service, so it exercises the actual upstream connections: the rotation must open a new +// upstream connection authenticated with the new key, close the demoted key's connection once the +// new anchor commits, and keep the downstream connection (authenticated with the old key) serving +// exactly one copy of each update. +func TestAutoConfigKeyRotationClosesOldClientAndDoesNotDuplicateUpdates(t *testing.T) { + oldKey := testAutoConfEnv1.SDKKey() + flagKey := "rotation-test-flag" + flagV1 := ldbuilders.NewFlagBuilder(flagKey).Version(1).On(false).Build() + flagV2 := ldbuilders.NewFlagBuilder(flagKey).Version(2).On(true).Build() + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + // The fake LD streaming service serves /all; both the original and the rotated SDK clients + // connect to it, and sdkStream.Enqueue pushes an event to every connected client. + putEvent := ldservices.NewServerSDKData().Flags(&flagV1).ToPutEvent() + sdkStreamHandler, sdkStream := ldservices.ServerSideStreamingServiceHandler(putEvent) + defer sdkStream.Close() + recordedSDKStreamHandler, sdkRequestsCh := httphelpers.RecordingHandler(sdkStreamHandler) + + initialACEvent := makeAutoConfPutEvent(testAutoConfEnv1) + acHandler, acStream := httphelpers.SSEHandler(&initialACEvent) + defer acStream.Close() + + // One upstream server plays both roles: the auto-config stream and the SDK streaming service. + // (The auto-config path literal matches autoConfigStreamPath in internal/autoconfig, which is + // not exported.) + upstreamHandler := httphelpers.HandlerForPath("/relay_auto_config", acHandler, recordedSDKStreamHandler) + + eventsHandler := httphelpers.HandlerWithStatus(202) + + httphelpers.WithServer(upstreamHandler, func(upstreamServer *httptest.Server) { + httphelpers.WithServer(eventsHandler, func(eventsServer *httptest.Server) { + config := testAutoConfDefaultConfig + config.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(upstreamServer.URL) + config.Events.EventsURI, _ = configtypes.NewOptURLAbsoluteFromString(eventsServer.URL) + + // A nil clientFactory means real SDK clients. + relay, err := newRelayInternal(config, relayInternalOptions{loggers: mockLog.Loggers}) + require.NoError(t, err) + defer relay.Close() + + helper := relayTestHelper{t: t, relay: relay} + helper.awaitEnvironment(testAutoConfEnv1.id) + + // The original anchor's client connects upstream with the original key. + initialStreamReq := helpers.RequireValue(t, sdkRequestsCh, time.Second*5) + assert.Equal(t, string(oldKey), initialStreamReq.Request.Header.Get("Authorization")) + + httphelpers.WithServer(relay, func(relayServer *httptest.Server) { + // Connect a downstream server-side SDK client authenticated with the ORIGINAL key. It + // stays connected across the rotation, since the old key remains valid for its grace period. + req, err := http.NewRequest("GET", relayServer.URL+"/all", nil) + require.NoError(t, err) + req.Header.Set("Authorization", string(oldKey)) + stream, err := eventsource.SubscribeWithRequestAndOptions(req, + eventsource.StreamOptionLogger(mockLog.Loggers.ForLevel(ldlog.Info))) + require.NoError(t, err) + defer stream.Close() + + initialPut := helpers.RequireValue(t, stream.Events, time.Second*5, "timed out waiting for initial put") + require.Equal(t, "put", initialPut.Event()) + + // Rotate: a new key becomes the anchor and the old key is demoted with a one-hour expiry + // (the backend's default-rotation shape). + modified := makeEnvWithModifiedSDKKey(testAutoConfEnv1) + modified.sdkKey.Expiring = envfactory.ExpiringKeyRep{ + Value: oldKey, + Timestamp: ldtime.UnixMillisNow() + ldtime.UnixMillisecondTime(time.Hour.Milliseconds()), + } + acStream.Enqueue(makeAutoConfPatchEvent(modified)) + + // The new anchor's client connects upstream with the new key and re-initializes the + // handed-over store, which republishes a put to connected downstream clients. + rotationStreamReq := helpers.RequireValue(t, sdkRequestsCh, time.Second*5) + assert.Equal(t, string(modified.SDKKey()), rotationStreamReq.Request.Header.Get("Authorization")) + rotationPut := helpers.RequireValue(t, stream.Events, time.Second*5, "timed out waiting for rotation put") + require.Equal(t, "put", rotationPut.Event()) + + // The demoted key's client must be torn down once the new anchor commits; before the fix it + // stayed connected until the key's expiry, double-broadcasting every update in the meantime. + require.Eventually(t, func() bool { + return mockLog.HasMessageMatch(ldlog.Info, "Closing LaunchDarkly client") + }, time.Second*2, time.Millisecond*20, "the demoted key's SDK client was not closed after the rotation") + + // Push one flag update upstream. Exactly one upstream client (the new anchor's) receives + // it, so connected downstream clients must receive exactly one patch. + flagV2JSON, err := json.Marshal(flagV2) + require.NoError(t, err) + sdkStream.Enqueue(httphelpers.SSEEvent{ + Event: "patch", + Data: fmt.Sprintf(`{"path": "/flags/%s", "data": %s}`, flagKey, flagV2JSON), + }) + + patch := helpers.RequireValue(t, stream.Events, time.Second*5, "timed out waiting for patch") + assert.Equal(t, "patch", patch.Event()) + assert.Contains(t, patch.Data(), flagKey) + if !helpers.AssertNoMoreValues(t, stream.Events, time.Millisecond*500, + "received a duplicate stream update after key rotation") { + t.FailNow() + } + }) + }) + }) +} From 98086acaa4d72d2aa85038f074e62dc86ceda9d4 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Thu, 16 Jul 2026 11:10:26 -0700 Subject: [PATCH 39/66] refactor(relayenv): rename GetStreamHandler param to avoid shadowing credential package (#753) --- internal/relayenv/env_context_impl.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 39d4971c..bb62f6ee 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -920,12 +920,12 @@ func (c *envContextImpl) GetLoggers() ldlog.Loggers { return c.loggers } -func (c *envContextImpl) GetStreamHandler(streamProvider streams.StreamProvider, credential credential.SDKCredential) http.Handler { +func (c *envContextImpl) GetStreamHandler(streamProvider streams.StreamProvider, cred credential.SDKCredential) http.Handler { // Build the handler on demand rather than storing one per (credential, provider): every handler in a // (filter, provider) slot is identical except for the credential-derived channel id, which we resolve // here from the request's already-authenticated credential. c.filterKey is immutable after // construction, so this needs no lock. - if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, credential)); h != nil { + if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, cred)); h != nil { return h } return http.HandlerFunc(invalidStreamHandler) From e6c3afc46e8f6a613f2e6934b290aae9c58ca57a Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Fri, 17 Jul 2026 12:17:52 -0700 Subject: [PATCH 40/66] test(relay): fix flaky autoconfig key-rotation e2e test (#754) Fixes a timing race in TestAutoConfigKeyRotationClosesOldClientAndDoesNotDuplicateUpdates that could hit 503 "client was not initialized" on the first downstream GET /all. --- relay/autoconfig_key_rotation_end_to_end_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/relay/autoconfig_key_rotation_end_to_end_test.go b/relay/autoconfig_key_rotation_end_to_end_test.go index 18f19731..fdd189e8 100644 --- a/relay/autoconfig_key_rotation_end_to_end_test.go +++ b/relay/autoconfig_key_rotation_end_to_end_test.go @@ -80,6 +80,15 @@ func TestAutoConfigKeyRotationClosesOldClientAndDoesNotDuplicateUpdates(t *testi initialStreamReq := helpers.RequireValue(t, sdkRequestsCh, time.Second*5) assert.Equal(t, string(oldKey), initialStreamReq.Request.Header.Get("Authorization")) + // awaitEnvironment only waits for the environment to be registered; the real SDK client is + // built asynchronously (startSDKClient runs in a goroutine), so wait for it to finish + // initializing before issuing the downstream request below. Otherwise the request can race the + // client install and get a 503 "client was not initialized". This mirrors waitForSuccessfulInit + // in relay_end_to_end_test.go, and the "Closing LaunchDarkly client" wait later in this test. + require.Eventually(t, func() bool { + return mockLog.HasMessageMatch(ldlog.Info, "Initialized LaunchDarkly client for") + }, time.Second*2, time.Millisecond*20, "the environment's SDK client did not finish initializing") + httphelpers.WithServer(relay, func(relayServer *httptest.Server) { // Connect a downstream server-side SDK client authenticated with the ORIGINAL key. It // stays connected across the rotation, since the old key remains valid for its grace period. From 737be4702c9707223d0930995dc49334707de462 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Fri, 17 Jul 2026 12:18:43 -0700 Subject: [PATCH 41/66] test(concurrent-keys): additional integration tests (#755) --- relay/concurrent_keys_auth_test.go | 345 +++++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) diff --git a/relay/concurrent_keys_auth_test.go b/relay/concurrent_keys_auth_test.go index d04b24e1..993980f3 100644 --- a/relay/concurrent_keys_auth_test.go +++ b/relay/concurrent_keys_auth_test.go @@ -19,22 +19,27 @@ package relay // (proving the anchor owns the only upstream connection). import ( + "encoding/json" + "net/http" "slices" "testing" "time" "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/api" "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "github.com/launchdarkly/ld-relay/v8/internal/filedata" "github.com/launchdarkly/ld-relay/v8/internal/relayenv" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/configsource" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" "github.com/launchdarkly/eventsource" "github.com/launchdarkly/go-configtypes" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" "github.com/launchdarkly/go-server-sdk-evaluation/v3/ldbuilders" "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" @@ -57,6 +62,9 @@ const ( // rotatedAnchorSDKKey is a brand-new SDK key that becomes the anchor when the anchor is rotated. rotatedAnchorSDKKey = config.SDKKey("sdk-rotated-anchor") + + // addedSDKKey is a brand-new non-anchor SDK key introduced by a patch that adds an array entry. + addedSDKKey = config.SDKKey("sdk-added") ) var multiKeyIdentifiers = relayenv.EnvIdentifiers{ @@ -537,3 +545,340 @@ func awaitCredentialRemoved(t *testing.T, relay *Relay, cred credential.SDKCrede return err != nil }, time.Second, 5*time.Millisecond, "credential was not removed from the accepted set") } + +// An SDK-key-only environment (no mobile key) initializes and authenticates. +// +// This is the regression lock for the escaped no-mobile-key rejection: an environment configured with +// only a server-side SDK key — MobKey/mobileKeys absent — must configure and authenticate downstream +// rather than being rejected as credential-short. Covered on both the RAC and offline paths. + +// sdkOnlyEnvRep builds a RAC/offline EnvironmentRep for a server-side-only environment: a single SDK +// key in sdkKeys[] and no mobile key (neither the singular mobKey nor a mobileKeys array). +func sdkOnlyEnvRep(version int) envfactory.EnvironmentRep { + return envfactory.EnvironmentRep{ + EnvID: multiKeyEnvID, + EnvKey: multiKeyIdentifiers.EnvKey, + EnvName: multiKeyIdentifiers.EnvName, + ProjKey: multiKeyIdentifiers.ProjKey, + ProjName: multiKeyIdentifiers.ProjName, + SDKKey: envfactory.SDKKeyRep{Value: anchorSDKKey}, + SDKKeys: []envfactory.ConcurrentKeyRep{{Key: "anchor-sdk", Value: string(anchorSDKKey)}}, + Version: version, + } +} + +// sdkOnlyArchiveEnv is the offline-mode equivalent of sdkOnlyEnvRep: an accepted SDK key set of one +// and an empty accepted mobile key set (no mobile key). +func sdkOnlyArchiveEnv() filedata.ArchiveEnvironment { + return filedata.ArchiveEnvironment{ + Params: envfactory.EnvironmentParams{ + EnvID: multiKeyEnvID, + SDKKey: anchorSDKKey, + AcceptedSDKKeys: []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}}, + AcceptedMobileKeys: []envfactory.AcceptedMobileKey{}, + Identifiers: multiKeyIdentifiers, + }, + SDKData: multiKeySDKData(), + } +} + +func TestConcurrentKeysRAC_SDKOnlyEnvironmentAuthenticates(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent(sdkOnlyEnvRep(1)) + autoConfTest(t, testAutoConfDefaultConfig, &putEvent, func(p autoConfTestParams) { + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) + + env := p.awaitEnvironment(multiKeyEnvID) + + // The SDK key and env ID authenticate; the environment simply has no mobile key. + p.assertSDKEndpointsAvailability(true, anchorSDKKey, "", multiKeyEnvID) + assert.Empty(t, env.GetAcceptedKeys().Mobile, "a server-side-only env must have no mobile keys") + }) +} + +func TestConcurrentKeysOffline_SDKOnlyEnvironmentAuthenticates(t *testing.T) { + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(sdkOnlyArchiveEnv()) + + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) + + env := p.awaitEnvironment(multiKeyEnvID) + + p.assertSDKEndpointsAvailability(true, anchorSDKKey, "", multiKeyEnvID) + assert.Empty(t, env.GetAcceptedKeys().Mobile, "a server-side-only env must have no mobile keys") + + // Flag data flows through the store that the anchor connection populates. + flags, err := env.GetStore().GetAll(ldstoreimpl.Features()) + require.NoError(t, err) + assert.NotEmpty(t, flags) + }) +} + +// A single patch that adds one non-anchor key and removes another, with the anchor held fixed. +// +// Because the anchor value does not change, there is no re-anchor: the sole upstream client keeps +// serving. The added entry starts routing, the removed entry stops, and a downstream stream that was +// open before the patch is left undisturbed. Uses a real (dummy) client + RAC mock so there is a live +// stream to observe (FakeLDClient never serves a stream body). +func TestConcurrentKeysRAC_ArrayPatchAddsAndRemovesNonAnchorKeys(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + racMock := configsource.NewRACMock(t, &putEvent) + + cfg := config.Config{AutoConfig: config.AutoConfigConfig{Key: testAutoConfKey}} + cfg.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(racMock.URL) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + relay, err := newRelayInternal(cfg, relayInternalOptions{ + loggers: mockLog.Loggers, + clientFactory: testclient.CreateDummyClient, + }) + require.NoError(t, err) + defer relay.Close() + + h := relayTestHelper{t: t, relay: relay} + env := h.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + // Hold an open downstream stream on the anchor while the non-anchor entries change around it. + req := sharedtest.BuildRequestWithAuth("GET", "/all", anchorSDKKey, nil) + sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) + + // One patch that adds a new non-anchor key (addedSDKKey) and removes the existing one + // (extraSDKKey), keeping the anchor and both mobile keys. + patch := multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-sdk", Value: string(anchorSDKKey)}, + {Key: "added-sdk", Value: string(addedSDKKey)}, + }, + defaultMobileKeyReps(), + 2, + ) + racMock.Send(configsource.MakeAutoConfigPatchEvent(patch)) + + // The added key routes and the removed key stops routing. + require.Eventually(t, func() bool { + _, errAdded := relay.getEnvironment(sdkauth.New(addedSDKKey)) + _, errRemoved := relay.getEnvironment(sdkauth.New(extraSDKKey)) + return errAdded == nil && errRemoved != nil + }, 5*time.Second, 5*time.Millisecond) + + // The anchor's open stream is undisturbed by the non-anchor add/remove. + assertStreamStaysOpen(t, eventCh, 500*time.Millisecond) + }) + + // The anchor never changed, so no re-anchor happened and the anchor still owns the connection. + assert.Equal(t, anchorSDKKey, env.GetAcceptedKeys().Anchor) + h.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + h.assertSDKEndpointsAvailability(true, addedSDKKey, "", "") + h.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") +} + +// A malformed offline payload preserves the previous credentials and does not reconnect. +// +// The offline handler validates the credential set with BuildAcceptedSet; a structurally malformed +// payload (here, the anchor is absent from sdkKeys[]) must be rejected without applying it: the +// previously-accepted credentials stay live and the environment is not torn down or recreated. Unlike +// the RAC path there is no live stream to reconnect, so "preserve, no reconnect" is the whole policy. +func TestConcurrentKeysOffline_MalformedPayloadPreservesCredentialsWithoutReconnect(t *testing.T) { + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + _ = p.awaitEnvironment(multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + + // Reload with a structurally malformed payload: the anchor (anchorSDKKey) is not present in the + // accepted SDK key set. The handler must preserve the previous set rather than apply this. + malformed := multiKeyArchiveEnv( + []envfactory.AcceptedSDKKey{{Value: extraSDKKey}}, + defaultAcceptedMobileKeys(), + ) + p.updateHandler.UpdateEnvironment(malformed) + + p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "Malformed credential payload for offline environment") + + // Previous credentials preserved: the malformed set was not applied, so every key that + // authenticated before still does — and the environment itself is intact (its env-ID endpoints + // still resolve rather than 404). The offline path has no live stream, so there is nothing to + // reconnect; preserving the prior accepted set is the whole policy. + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + }) +} + +// Removing a key's expiry in a later payload cancels the scheduled drop. +// +// A non-anchor key given a future expiry becomes deprecated-but-accepted and is scheduled to be dropped +// when the expiry passes. If a later payload carries the same key with no expiry, the reconcile refreshes +// its metadata back to permanent: it leaves the deprecated set and the cleanup ticker never drops it. +func TestConcurrentKeysRAC_DeExpiryCancelsScheduledDrop(t *testing.T) { + cfg := testAutoConfDefaultConfig + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + autoConfTest(t, cfg, &putEvent, func(p autoConfTestParams) { + _ = p.awaitClient() + env := p.awaitEnvironment(multiKeyEnvID) + + // Give the non-anchor SDK key a far-future expiry: it becomes deprecated-but-accepted with a drop + // scheduled for the expiry. (Far-future so the drop cannot fire during the test — we're proving the + // de-expiry cancels it, not that the key survives its own deadline.) + expiry := time.Now().Add(1 * time.Hour).UnixMilli() + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-sdk", Value: string(anchorSDKKey)}, + {Key: "extra-sdk", Value: string(extraSDKKey), Expiry: msPtr(expiry)}, + }, + defaultMobileKeyReps(), + 2, + ))) + require.Eventually(t, func() bool { return credsContain(env.GetDeprecatedCredentials(), extraSDKKey) }, + time.Second, 5*time.Millisecond, "expiry was not applied — nothing to cancel") + + // A later payload carries the key with no expiry: the scheduled drop is cancelled. + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 3))) + + require.Eventually(t, func() bool { return !credsContain(env.GetDeprecatedCredentials(), extraSDKKey) }, + time.Second, 5*time.Millisecond, "de-expiry did not return the key to the permanent set") + + // The key is permanent again: its expiry is cleared, so the cleanup ticker has nothing to drop. + info, ok := env.GetAcceptedKeys().Server[extraSDKKey] + require.True(t, ok, "the de-expired key must still be accepted") + assert.Nil(t, info.Expiry, "de-expiry must clear the scheduled drop (the expiry returns to permanent)") + + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + }) +} + +// Renaming a key's identifier (same value, new "key") disturbs no credential and updates the status. +// +// Credential identity is by value, so changing only the wire "key" identifier neither drops nor re-adds +// the credential (no new upstream client, uninterrupted authentication). The reconcile does refresh the +// stored identifier, so the status endpoint surfaces the new name in sdkKeys[]. This holds whether the +// renamed key is a non-anchor or the anchor itself: the anchor is selected by value, so renaming its +// identifier — while its value stays put — is a plain rename, not a re-anchor. +func TestConcurrentKeysRAC_RenamePreservesCredentialAndUpdatesStatusIdentifier(t *testing.T) { + const renamedID = "sdk-renamed" + tests := []struct { + name string + renamedKey config.SDKKey + }{ + {"non-anchor key", extraSDKKey}, + {"anchor key", anchorSDKKey}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + autoConfTest(t, testAutoConfDefaultConfig, &putEvent, func(p autoConfTestParams) { + _ = p.awaitClient() + env := p.awaitEnvironment(multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + + // Rebuild the sdkKeys array with the target key's identifier changed. The anchor's value + // (sdkKey.value) is left as anchorSDKKey either way, so this is a rename, not a re-anchor. + sdkKeys := defaultSDKKeyReps() + for i := range sdkKeys { + if config.SDKKey(sdkKeys[i].Value) == tt.renamedKey { + sdkKeys[i].Key = renamedID + } + } + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(multiKeyEnvRep(sdkKeys, defaultMobileKeyReps(), 2))) + + // The identifier is refreshed in place — the credential itself is untouched. + require.Eventually(t, func() bool { + info, ok := env.GetAcceptedKeys().Server[tt.renamedKey] + return ok && info.Key != nil && *info.Key == renamedID + }, time.Second, 5*time.Millisecond, "the renamed identifier was not applied") + + // No re-anchor and no credential churn: the anchor is unchanged, no new client is built, + // and both keys keep authenticating. + assert.Equal(t, anchorSDKKey, env.GetAcceptedKeys().Anchor) + p.shouldNotCreateClient(200 * time.Millisecond) + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + + // The status endpoint reflects the new identifier for that key. + req, _ := http.NewRequest("GET", "/status", nil) + result, body := sharedtest.DoRequest(req, p.relay) + require.Equal(t, http.StatusOK, result.StatusCode) + var status api.StatusRep + require.NoError(t, json.Unmarshal(body, &status)) + require.Len(t, status.Environments, 1) + var envStatus api.EnvironmentStatusRep + for _, e := range status.Environments { + envStatus = e + } + renamed := findSDKKeyStatus(envStatus.SDKKeys, sdks.ObscureKey(string(tt.renamedKey))) + require.NotNil(t, renamed, "renamed key not present in status sdkKeys[]") + assert.Equal(t, renamedID, renamed.Key) + }) + }) + } +} + +// findSDKKeyStatus returns the sdkKeys[] entry whose obscured value matches, or nil if absent. +func findSDKKeyStatus(keys []api.KeyStatus, obscuredValue string) *api.KeyStatus { + for i := range keys { + if keys[i].Value == obscuredValue { + return &keys[i] + } + } + return nil +} + +// A single payload that adds a key, re-anchors, and removes a key all at once. +// +// The operations apply in order add -> re-anchor -> remove, so the end state is deterministic: the new +// anchor opens the sole upstream client and the old anchor's client closes; the added key is accepted; +// the old anchor and the removed non-anchor key are gone. Runs on the FakeLDClient harness, which +// verifies the routing/credential-level outcome of the swap (the real-upstream store handover is +// exercised by the re-anchor tests in the relayenv package). +func TestConcurrentKeysRAC_MixedUpdateAddsReanchorsAndRemovesInOnePayload(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + autoConfTest(t, testAutoConfDefaultConfig, &putEvent, func(p autoConfTestParams) { + client1 := p.awaitClient() + assert.Equal(t, anchorSDKKey, client1.Key) + _ = p.awaitEnvironment(multiKeyEnvID) + + // One patch: add addedSDKKey, re-anchor to rotatedAnchorSDKKey (brand-new), and drop extraSDKKey. + mixed := envfactory.EnvironmentRep{ + EnvID: multiKeyEnvID, + EnvKey: multiKeyIdentifiers.EnvKey, + EnvName: multiKeyIdentifiers.EnvName, + ProjKey: multiKeyIdentifiers.ProjKey, + ProjName: multiKeyIdentifiers.ProjName, + SDKKey: envfactory.SDKKeyRep{Value: rotatedAnchorSDKKey}, + MobKey: anchorMobileKey, + SDKKeys: []envfactory.ConcurrentKeyRep{ + {Key: "rotated-anchor-sdk", Value: string(rotatedAnchorSDKKey)}, + {Key: "added-sdk", Value: string(addedSDKKey)}, + }, + MobileKeys: defaultMobileKeyReps(), + Version: 2, + } + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(mixed)) + + // Re-anchor: the new anchor opens the single upstream client; the old anchor's client closes and no + // additional client is created for the added non-anchor key. + client2 := p.awaitClient() + assert.Equal(t, rotatedAnchorSDKKey, client2.Key) + client1.AwaitClose(t, 5*time.Second) + p.shouldNotCreateClient(200 * time.Millisecond) + + awaitCredentialRemoved(t, p.relay, anchorSDKKey) + awaitCredentialRemoved(t, p.relay, extraSDKKey) + + // End state: new anchor + added key authenticate; old anchor + removed key do not. + p.assertSDKEndpointsAvailability(true, rotatedAnchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, addedSDKKey, "", "") + p.assertSDKEndpointsAvailability(false, anchorSDKKey, "", "") + p.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") + }) +} From da81f6e5f8cc0a9267bd49a4aa72bccc0697cb37 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Sun, 19 Jul 2026 20:14:36 -0700 Subject: [PATCH 42/66] docs(concurrent-keys): record as-shipped re-anchor and validation semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The final pre-merge review found five places where the implementation deliberately moved past the design text without the doc being updated. Recorded as implementation notes rather than rewrites so the original reasoning stays visible: - §7: demoted anchor's upstream client closes at commit (double-broadcast avoidance); Case B re-promotion builds fresh; synchronous build occupies the RAC dispatch goroutine (operational characteristic). - §7/§8: rollback on re-anchor init failure is scoped to the anchor change; other payload changes stand; recovery requires a version-bumped update. Resolves the §8 open question on atomicity. - §9: validation runs at the parse boundary before the version is recorded, and is stricter than the two originally named cases. - §6.3: the legacy expiring slot is read for old-format payloads only. --- .agent-docs/concurrent-keys/phase1-design.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.agent-docs/concurrent-keys/phase1-design.md b/.agent-docs/concurrent-keys/phase1-design.md index 1ac645b4..6e36bf17 100644 --- a/.agent-docs/concurrent-keys/phase1-design.md +++ b/.agent-docs/concurrent-keys/phase1-design.md @@ -232,6 +232,8 @@ On default rotation the backend mirrors expiry info into both: **Decision**: new relays trust the array. The legacy `sdkKey.expiring{}` field is treated as a write-only back-compat shim — new relays do not read it. (Working assumption pending team confirmation.) +**Precision (as shipped):** "do not read it" applies when the arrays are present. For an *old-format* payload (no `sdkKeys[]`), the legacy slot is the only source for the deprecated key, and relay does read it there — synthesizing it into the accepted set with its expiry. The rule as implemented: arrays present ⇒ legacy slot ignored; arrays absent ⇒ legacy slot honored. + --- ## 7. Re-anchoring @@ -253,6 +255,12 @@ This is the highest-risk piece of Phase 1. The **T0 PoC** validated the swap mec This order is *necessary* — the PoC found that flipping the pointer too early leaves `GetClient()` nil mid-swap (H6) and breaks the env on init failure (H7) — and *sufficient* only with the store-handling approach below. +> **Implementation notes — as shipped (recorded at final review, 2026-07-18):** +> +> - **Step 6 as implemented:** the demoted anchor's *upstream client* is closed at commit time, not after the grace period. With store handover, a second live upstream client would double-broadcast every update into the shared store wrapper. The demoted key's *credential mappings* stay registered until its grace expires, so downstream SDKs authenticating with it keep working through the window — only the upstream connection goes early (`removeCredential` later finds no client to close). +> - **Case B in practice:** because the demoted client is closed at commit, a re-promoted former anchor has no live client and takes the build path. The "reuse a live client" branch survives defensively; `NewAnchorPreviouslyAccepted` governs credential-mapping registration only. +> - **Operational characteristic:** the synchronous client build runs on the shared RAC dispatch goroutine. While one environment re-anchors, other environments' config updates (and the expiry tickers, which share the reconcile lock) wait — up to `InitTimeout` (default 10s) per re-anchoring environment in the worst case. Data-plane serving is unaffected throughout. This is the accepted cost of build-before-flip; a mass rotation across N environments serializes to roughly N × build time on the config plane. + ### Two re-anchor cases — Case A (new key) vs Case B (already-accepted key) The sequence above is **Case A**: the new anchor is a key relay has not previously accepted, so no SDK client exists for it. Relay must build one, hand over the store, wait for `Initialized()`, then flip and re-wire. @@ -295,7 +303,9 @@ This is the concrete form of decoupling the store's lifecycle from the client's. ### Failure handling -If the new client fails to initialize, the swap **rolls back**: the rotator's anchor pointer stays on the old key (the caller simply does not call `CommitAnchor`), the previous accepted set is preserved, and a structured error is logged. The old anchor's client (still alive in its grace period) continues to serve. This is the §8 atomicity principle applied to re-anchor, and it applies to **Case A only** — Case B reuses an already-initialized client and has nothing to fail. Relay has no dedicated alarm infrastructure today; an `Error`-level structured log (`globalLoggers.Errorf`) is the strongest signal available and is sufficient. +If the new client fails to initialize, the swap **rolls back**: the rotator's anchor pointer stays on the old key (the caller simply does not call `CommitAnchor`), the anchor-related changes are reverted — the previous anchor is re-admitted and kept serving even if the payload revoked it outright, and a brand-new failed anchor is dropped — and a structured error is logged. The old anchor's client continues to serve. It applies to **Case A only** — Case B reuses an already-initialized client and has nothing to fail. Relay has no dedicated alarm infrastructure today; an `Error`-level structured log (`globalLoggers.Errorf`) is the strongest signal available and is sufficient. + +**As-shipped rollback scope (recorded at final review, 2026-07-18):** the rollback is scoped to the *anchor change*, not the whole reconcile — other credential changes in the same payload (adds and removals of non-anchor keys) stand. **Recovery:** the payload's version was already recorded at the stream parse boundary, so an identical retry — or a reconnect's fresh `put` at the same version — is deduplicated; the environment stays on the previous anchor until the backend sends a *version-bumped* update. ### Consolidated specification for T2.c / T2.d @@ -333,7 +343,7 @@ This order ensures the accepted set is a *superset* during the transition. The n ### Atomicity -Reconcile is **all-or-nothing**. On partial failure (malformed payload, new-client init failure, etc.), log a structured error and preserve the previous accepted set. Working assumption — open question for the team. Aligns with the malformed-payload policy (§9). +**As shipped (this resolves the earlier open question):** a **malformed payload** is all-or-nothing — it is rejected at the stream parse boundary before any state mutation, so the previous accepted set is fully preserved (§9). A **re-anchor init failure** rolls back the anchor change only; the payload's other adds and removals stand (§7 failure handling). Full-reconcile rollback was considered and rejected: it would add snapshot/restore machinery across mappings and streams for no clear benefit given trusted sources, and the partial semantics are strictly safer for the non-anchor keys involved (valid new keys start working; revoked keys stay revoked; the anchor never breaks). ### Edge cases @@ -356,6 +366,8 @@ When relay receives a malformed RAC payload — most importantly, `sdkKey.value` This is the same atomicity principle as §8, applied at the boundary between trusted-source input and relay's internal state, with the added piece (reconnect) needed because RAC has no acknowledgment mechanism for failed-payload rejection. +**Implementation notes (recorded at final review, 2026-07-18):** validation runs at the stream parse boundary, *before* the message's version is recorded — this is what makes the forced fresh `put` (which carries the same version) re-processable rather than deduplicated away. The shipped validation is also stricter than the two cases named above: array entries with empty `value`s, and a defined `mobKey` absent from `mobileKeys[]` (the mobile analogue of the anchor invariant), are also rejected as malformed. + --- ## 10. Backwards compatibility From 153b78e3bc231f4cc7d54c7135293d6064e7009d Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 21 Jul 2026 10:59:04 -0700 Subject: [PATCH 43/66] fix(store): prevent a startup race from leaving the environment with a closed data store (#759) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes a startup race where the initial anchor client’s Build and a concurrent re-anchor Build could each create a wrapper; the last install could win, and closing the superseded client would tear down the store the environment still served. --- internal/store/relay_feature_store.go | 9 +- internal/store/store_concurrent_build_test.go | 100 ++++++++++++++++++ 2 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 internal/store/store_concurrent_build_test.go diff --git a/internal/store/relay_feature_store.go b/internal/store/relay_feature_store.go index b4805c84..327bac95 100644 --- a/internal/store/relay_feature_store.go +++ b/internal/store/relay_feature_store.go @@ -78,14 +78,16 @@ func NewSSERelayDataStoreAdapter( func (a *SSERelayDataStoreAdapter) Build( context subsystems.ClientContext, ) (subsystems.DataStore, error) { + // The lock is held across the whole build so two concurrent Build calls cannot each construct and + // install their own wrapper. a.mu.Lock() + defer a.mu.Unlock() + if existing := a.store; existing != nil { if sw, ok := existing.(*streamUpdatesStoreWrapper); ok && sw.acquire() { - a.mu.Unlock() return sw, nil } } - a.mu.Unlock() wrappedStore, err := a.wrappedFactory.Build(context) if err != nil { @@ -96,9 +98,6 @@ func (a *SSERelayDataStoreAdapter) Build( wrappedStore, context.GetLogging().Loggers, ) - - a.mu.Lock() - defer a.mu.Unlock() a.store = sw return sw, nil } diff --git a/internal/store/store_concurrent_build_test.go b/internal/store/store_concurrent_build_test.go new file mode 100644 index 00000000..6d4fd28c --- /dev/null +++ b/internal/store/store_concurrent_build_test.go @@ -0,0 +1,100 @@ +package store + +// Regression test for a race in SSERelayDataStoreAdapter.Build: two Build calls that both observe +// no existing store — the environment's initial anchor-client build racing the first re-anchor's +// synchronous build — must not each construct and install their own wrapper. If they did, the last +// writer would win adapter.store, and when that writer's client is later discarded as superseded, its +// Close would tear down the store the adapter is serving — evaluations and stream queries would read +// a closed store while the live upstream client fed one nothing reads. Build holds the adapter lock +// across the whole build, so the two calls are serialized: the first installs its wrapper and the +// second adopts that same wrapper via the fast path rather than building its own. +// +// The realistic trigger is a persistent store whose construction is slow at startup (e.g. Redis +// briefly unreachable) while a rotation patch re-anchors the environment. + +import ( + "sync/atomic" + "testing" + "time" + + "github.com/launchdarkly/go-server-sdk/v7/subsystems" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// gatedStoreFactory stalls its first Build call inside the wrapped factory until released, +// modelling a slow persistent-store connection at startup. Later calls proceed immediately. +type gatedStoreFactory struct { + inner countingCloseStoreFactory + entered chan struct{} + release chan struct{} + calls atomic.Int32 +} + +func (f *gatedStoreFactory) Build(ctx subsystems.ClientContext) (subsystems.DataStore, error) { + if f.calls.Add(1) == 1 { + f.entered <- struct{}{} + <-f.release + } + return f.inner.Build(ctx) +} + +func TestConcurrentBuildSerializesAndSharesOneWrapper(t *testing.T) { + factory := &gatedStoreFactory{entered: make(chan struct{}, 1), release: make(chan struct{})} + adapter := NewSSERelayDataStoreAdapter(factory, &mockEnvStreamsUpdates{}) + + // The initial anchor client's build stalls inside the wrapped factory. Because Build holds the + // adapter lock across the whole build, it holds the lock for the duration of this stall. + firstResult := make(chan subsystems.DataStore, 1) + go func() { + sw, err := adapter.Build(subsystems.BasicClientContext{}) + assert.NoError(t, err) + firstResult <- sw + }() + <-factory.entered + + // A re-anchor's synchronous build starts while the first is stalled. It must block on the adapter + // lock — it cannot build and install its own wrapper. + secondResult := make(chan subsystems.DataStore, 1) + go func() { + sw, err := adapter.Build(subsystems.BasicClientContext{}) + assert.NoError(t, err) + secondResult <- sw + }() + + select { + case <-secondResult: + t.Fatal("the second build returned while the first still held the lock; builds were not serialized") + case <-time.After(100 * time.Millisecond): + } + + // Release the stalled build. It installs its wrapper; the second build then adopts that same + // wrapper via the fast path rather than building its own. + close(factory.release) + firstStore := <-firstResult + secondStore := <-secondResult + + require.Same(t, firstStore, secondStore, "both builds must share the one installed wrapper") + require.Same(t, firstStore, adapter.GetStore()) + require.Equal(t, int32(1), factory.calls.Load(), "the wrapped factory must be built exactly once") + + built := factory.inner.allBuilt() + require.Len(t, built, 1, "no discarded second wrapper was ever constructed") + + // The first build's client is later discarded as superseded and closed. That releases one handover + // reference; the adapter keeps serving an open store. + require.NoError(t, firstStore.Close()) + require.Same(t, secondStore, adapter.GetStore()) + sw, ok := adapter.GetStore().(*streamUpdatesStoreWrapper) + require.True(t, ok) + sw.refMu.Lock() + closed := sw.closed + sw.refMu.Unlock() + require.False(t, closed, "the adapter must not be serving a torn-down store") + assert.Equal(t, 0, built[0].closeCount(), "the underlying store remains open while a holder remains") + + // The final holder's release (environment teardown) closes the underlying store exactly once. + require.NoError(t, secondStore.Close()) + assert.Equal(t, 1, built[0].closeCount()) +} From e96127d7f82d5be681c5235c53db3fbba5877b75 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 21 Jul 2026 11:00:00 -0700 Subject: [PATCH 44/66] refactor(concurrent-keys): remove dead code identified in final review (#760) Removes dead and redundant credential plumbing left over after the concurrent-keys / accepted-set model, without changing wire parsing for old sdkKey.expiring payloads (that path still synthesizes AcceptedSDKKeys in ToParams). --- internal/autoconfig/stream_manager.go | 2 +- internal/credential/credential.go | 11 ---- internal/credential/rotator.go | 6 -- internal/envfactory/env_params.go | 13 ---- internal/envfactory/env_rep.go | 20 ++---- internal/envfactory/env_rep_test.go | 6 +- internal/envfactory/reconcile_helper.go | 19 +++--- internal/envfactory/reconcile_helper_test.go | 68 +++++--------------- internal/relayenv/env_context_impl.go | 48 ++++---------- relay/autoconfig_actions.go | 5 +- relay/autoconfig_actions_test.go | 2 +- relay/filedata_actions.go | 4 +- relay/filedata_actions_test.go | 12 ++-- relay/filedata_testdata_test.go | 8 +-- 14 files changed, 56 insertions(+), 168 deletions(-) diff --git a/internal/autoconfig/stream_manager.go b/internal/autoconfig/stream_manager.go index 28c25871..2474aa1e 100644 --- a/internal/autoconfig/stream_manager.go +++ b/internal/autoconfig/stream_manager.go @@ -507,7 +507,7 @@ func (s *StreamManager) dispatchEnvAction(id config.EnvironmentID, rep envfactor // not advanced, so the fresh put — which carries the same version — is not deduplicated away by the // MessageReceiver. Any error from BuildAcceptedSet is a *MalformedCredentialSetError. func (s *StreamManager) validateCredentialPayload(rep envfactory.EnvironmentRep) error { - _, _, err := envfactory.BuildAcceptedSet(rep.ToParams()) + _, err := envfactory.BuildAcceptedSet(rep.ToParams()) return err } diff --git a/internal/credential/credential.go b/internal/credential/credential.go index 09354588..dc696378 100644 --- a/internal/credential/credential.go +++ b/internal/credential/credential.go @@ -14,14 +14,3 @@ type SDKCredential interface { // Masked returns a masked form of the credential suitable for log messages. Masked() string } - -// AutoConfig represents credentials that are updated via AutoConfig protocol. -type AutoConfig struct { - // SDKKey is the environment's SDK key; if there is more than one active key, it is the latest. - SDKKey SDKCredential - // ExpiringSDKKey is an additional SDK key that may or may not be present; it represents the fact that a deprecated - // key may exist which can still authenticate a given connection. - ExpiringSDKKey SDKCredential - // MobileKey is the environment's mobile key. - MobileKey SDKCredential -} diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 558844d6..2b202bd2 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -62,12 +62,6 @@ type Rotator struct { mu sync.RWMutex } -type InitialCredentials struct { - SDKKey config.SDKKey - MobileKey config.MobileKey - EnvironmentID config.EnvironmentID -} - // NewRotator constructs a rotator with the provided loggers. A new rotator // contains no credentials and can optionally be initialized via Initialize. func NewRotator(loggers ldlog.Loggers) *Rotator { diff --git a/internal/envfactory/env_params.go b/internal/envfactory/env_params.go index a0d02a0a..35acffcd 100644 --- a/internal/envfactory/env_params.go +++ b/internal/envfactory/env_params.go @@ -26,10 +26,6 @@ type EnvironmentParams struct { // MobileKey is the environment's mobile key. MobileKey config.MobileKey - // ExpiringSDKKey is an additional SDK key that should also be allowed (but not surfaced as - // the canonical one). - ExpiringSDKKey ExpiringSDKKey - // AcceptedSDKKeys is the full accepted set of SDK keys for this environment, including the // anchor. Always non-nil after ToParams(): non-empty sdkKeys arrays populate directly; absent // or empty sdkKeys are synthesized from the singular sdkKey field so there is always at least @@ -64,15 +60,6 @@ type AcceptedMobileKey struct { Expiry time.Time } -type ExpiringSDKKey struct { - Key config.SDKKey - Expiration time.Time -} - -func (e ExpiringSDKKey) Defined() bool { - return e.Key.Defined() -} - func (e EnvironmentParams) WithFilter(key config.FilterKey) EnvironmentParams { e.Identifiers.FilterKey = key return e diff --git a/internal/envfactory/env_rep.go b/internal/envfactory/env_rep.go index 151f93a2..d83f7ec0 100644 --- a/internal/envfactory/env_rep.go +++ b/internal/envfactory/env_rep.go @@ -110,17 +110,6 @@ type ConcurrentKeyRep struct { Expiry *int64 `json:"expiry,omitempty"` // Unix-ms; nil = permanent } -func (e ExpiringKeyRep) ToParams() ExpiringSDKKey { - if e.Value.Defined() { - return ExpiringSDKKey{ - Key: e.Value, - Expiration: ToTime(e.Timestamp), - } - } else { - return ExpiringSDKKey{} - } -} - func ToTime(millisecondTime ldtime.UnixMillisecondTime) time.Time { return time.UnixMilli(int64(millisecondTime)) //nolint: gosec } @@ -135,11 +124,10 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { ProjKey: r.ProjKey, ProjName: r.ProjName, }, - SDKKey: r.SDKKey.Value, - ExpiringSDKKey: r.SDKKey.Expiring.ToParams(), - MobileKey: r.MobKey, - TTL: time.Duration(r.DefaultTTL) * time.Minute, - SecureMode: r.SecureMode, + SDKKey: r.SDKKey.Value, + MobileKey: r.MobKey, + TTL: time.Duration(r.DefaultTTL) * time.Minute, + SecureMode: r.SecureMode, } if len(r.SDKKeys) > 0 { diff --git a/internal/envfactory/env_rep_test.go b/internal/envfactory/env_rep_test.go index 1d48e237..73c8e216 100644 --- a/internal/envfactory/env_rep_test.go +++ b/internal/envfactory/env_rep_test.go @@ -66,11 +66,7 @@ func TestEnvironmentRepToParams(t *testing.T) { ProjKey: "projkey2", ProjName: "projname2", }, - SDKKey: env2.SDKKey.Value, - ExpiringSDKKey: ExpiringSDKKey{ - Key: env2.SDKKey.Expiring.Value, - Expiration: time.UnixMilli(int64(env2.SDKKey.Expiring.Timestamp)), - }, + SDKKey: env2.SDKKey.Value, MobileKey: env2.MobKey, AcceptedSDKKeys: []AcceptedSDKKey{ {Value: env2.SDKKey.Value}, diff --git a/internal/envfactory/reconcile_helper.go b/internal/envfactory/reconcile_helper.go index 0f117736..28f8d426 100644 --- a/internal/envfactory/reconcile_helper.go +++ b/internal/envfactory/reconcile_helper.go @@ -1,13 +1,12 @@ package envfactory import ( - "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/util" ) -// BuildAcceptedSet converts an EnvironmentParams into the AcceptedSet and anchor -// credential needed by EnvContext.ReconcileCredentials. +// BuildAcceptedSet converts an EnvironmentParams into the AcceptedSet needed by +// EnvContext.ReconcileCredentials. // // Credential identity is keyed by value (the secret string), not by key (the human-readable // identifier). A rename — same value, different identifier — therefore produces the same @@ -26,7 +25,7 @@ import ( // is absent from params.AcceptedSDKKeys, or an array entry with an empty value. The caller must // preserve the previous accepted state and, for RAC handlers, reconnect the stream with jitter to // force a fresh put. This is the single home for the anchor invariant. -func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config.SDKKey, error) { +func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, error) { anchor := params.SDKKey b := credential.NewAcceptedSetBuilder().WithEnvironmentID(params.EnvID) @@ -40,7 +39,7 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. anchorInArray := false for _, k := range params.AcceptedSDKKeys { if !k.Value.Defined() { - return credential.AcceptedSet{}, anchor, credential.NewEmptyCredentialError("sdkKeys", k.Key) + return credential.AcceptedSet{}, credential.NewEmptyCredentialError("sdkKeys", k.Key) } if k.Value == anchor { anchorInArray = true @@ -54,7 +53,7 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. // synthesizes it into the array for old-format payloads). A defined anchor absent from the array is // a structurally malformed payload — reject it. if anchor.Defined() && !anchorInArray { - return credential.AcceptedSet{}, anchor, credential.NewAnchorNotInSetError() + return credential.AcceptedSet{}, credential.NewAnchorNotInSetError() } // Add every accepted mobile key, designating the primary as we encounter it. Like the anchor, @@ -63,7 +62,7 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. primaryMobileInArray := false for _, k := range params.AcceptedMobileKeys { if !k.Value.Defined() { - return credential.AcceptedSet{}, anchor, credential.NewEmptyCredentialError("mobileKeys", k.Key) + return credential.AcceptedSet{}, credential.NewEmptyCredentialError("mobileKeys", k.Key) } if k.Value == params.MobileKey { primaryMobileInArray = true @@ -78,12 +77,12 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, config. // without this guard the primary would be silently left undesignated, clearing it on reconcile and // breaking event forwarding. (An undefined mobKey is valid — a server-side-only environment.) if params.MobileKey.Defined() && !primaryMobileInArray { - return credential.AcceptedSet{}, anchor, credential.NewPrimaryMobileKeyNotInSetError() + return credential.AcceptedSet{}, credential.NewPrimaryMobileKeyNotInSetError() } set, err := b.Build() if err != nil { - return credential.AcceptedSet{}, anchor, err + return credential.AcceptedSet{}, err } - return set, anchor, nil + return set, nil } diff --git a/internal/envfactory/reconcile_helper_test.go b/internal/envfactory/reconcile_helper_test.go index 97479490..a5b63cad 100644 --- a/internal/envfactory/reconcile_helper_test.go +++ b/internal/envfactory/reconcile_helper_test.go @@ -50,11 +50,9 @@ func TestBuildAcceptedSet_HappyPath(t *testing.T) { []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, "mob-primary", ) - set, anchor, err := BuildAcceptedSet(params) + set, err := BuildAcceptedSet(params) require.NoError(t, err) - assert.Equal(t, config.SDKKey("sdk-anchor"), anchor) - expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). @@ -74,11 +72,9 @@ func TestBuildAcceptedSet_MultipleKeys(t *testing.T) { }, "mob-primary", ) - set, anchor, err := BuildAcceptedSet(params) + set, err := BuildAcceptedSet(params) require.NoError(t, err) - assert.Equal(t, config.SDKKey("sdk-anchor"), anchor) - expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). @@ -103,8 +99,8 @@ func TestBuildAcceptedSet_Rename(t *testing.T) { "mob-primary", ) - setOld, _, errOld := BuildAcceptedSet(paramsOldName) - setNew, _, errNew := BuildAcceptedSet(paramsNewName) + setOld, errOld := BuildAcceptedSet(paramsOldName) + setNew, errNew := BuildAcceptedSet(paramsNewName) require.NoError(t, errOld) require.NoError(t, errNew) @@ -147,8 +143,8 @@ func TestBuildAcceptedSet_Deexpiry(t *testing.T) { "mob-primary", ) - setWithExpiry, _, errWithExpiry := BuildAcceptedSet(paramsWithExpiry) - setNoExpiry, _, errNoExpiry := BuildAcceptedSet(paramsNoExpiry) + setWithExpiry, errWithExpiry := BuildAcceptedSet(paramsWithExpiry) + setNoExpiry, errNoExpiry := BuildAcceptedSet(paramsNoExpiry) require.NoError(t, errWithExpiry) require.NoError(t, errNoExpiry) @@ -177,7 +173,7 @@ func TestBuildAcceptedSet_AnchorNotInArray(t *testing.T) { }, "mob-primary", ) - _, _, err := BuildAcceptedSet(params) + _, err := BuildAcceptedSet(params) require.Error(t, err) var malformed *credential.MalformedCredentialSetError @@ -198,7 +194,7 @@ func TestBuildAcceptedSet_PrimaryMobileNotInArray(t *testing.T) { {Key: "other", Value: "mob-other"}, // ...but NOT in the array }, } - _, _, err := BuildAcceptedSet(params) + _, err := BuildAcceptedSet(params) require.Error(t, err) var malformed *credential.MalformedCredentialSetError @@ -215,11 +211,9 @@ func TestBuildAcceptedSet_NoMobileKey(t *testing.T) { SDKKey: SDKKeyRep{Value: config.SDKKey("sdk-anchor")}, // no MobKey, no MobileKeys } - set, anchor, err := BuildAcceptedSet(rep.ToParams()) + set, err := BuildAcceptedSet(rep.ToParams()) require.NoError(t, err) - assert.Equal(t, config.SDKKey("sdk-anchor"), anchor) - expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor"})) @@ -236,7 +230,7 @@ func TestBuildAcceptedSet_AnchorUndefined(t *testing.T) { }, "mob-primary", ) - _, _, err := BuildAcceptedSet(params) + _, err := BuildAcceptedSet(params) require.Error(t, err) var malformed *credential.MalformedCredentialSetError @@ -252,7 +246,7 @@ func TestBuildAcceptedSet_NoSDKKeys(t *testing.T) { AcceptedSDKKeys: []AcceptedSDKKey{}, AcceptedMobileKeys: []AcceptedMobileKey{}, } - _, _, err := BuildAcceptedSet(params) + _, err := BuildAcceptedSet(params) require.Error(t, err, "a set with no SDK key at all must be rejected") } @@ -275,11 +269,9 @@ func TestBuildAcceptedSet_MixedUpdate(t *testing.T) { }, "mob-primary", ) - set, anchor, err := BuildAcceptedSet(params) + set, err := BuildAcceptedSet(params) require.NoError(t, err) - assert.Equal(t, config.SDKKey("sdk-new-anchor"), anchor) - expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). WithAnchor(credential.SDKKeyParams{Value: "sdk-new-anchor", Key: util.PtrOrNil("new-default")}). @@ -301,11 +293,9 @@ func TestBuildAcceptedSet_AnchorNeverExpiring(t *testing.T) { }, "mob-primary", ) - set, anchor, err := BuildAcceptedSet(params) + set, err := BuildAcceptedSet(params) require.NoError(t, err) - assert.Equal(t, config.SDKKey("sdk-anchor"), anchor) - // Anchor is permanent (WithAnchor), not expiring — identical to a payload with no anchor expiry. expected := mustBuild(t, credential.NewAcceptedSetBuilder(). WithEnvironmentID("env-abc"). @@ -329,7 +319,7 @@ func TestBuildAcceptedSet_MultipleMobileKeys(t *testing.T) { {Key: "mob-2", Value: "mob-secondary"}, }, } - set, _, err := BuildAcceptedSet(params) + set, err := BuildAcceptedSet(params) require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). @@ -355,7 +345,7 @@ func TestBuildAcceptedSet_ExpiringMobileKey(t *testing.T) { {Key: "mob-old", Value: "mob-old", Expiry: expiry1}, // expiring }, } - set, _, err := BuildAcceptedSet(params) + set, err := BuildAcceptedSet(params) require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). @@ -365,31 +355,3 @@ func TestBuildAcceptedSet_ExpiringMobileKey(t *testing.T) { WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary", Key: util.PtrOrNil("mob-1")})) assert.Equal(t, expected, set, "expiring mobile key must land as an expiring key in the set") } - -// TestBuildAcceptedSet_TrustTheArray verifies that the legacy sdkKey.expiring slot is not -// consulted: when EnvironmentParams.ExpiringSDKKey is populated (from the legacy field) but -// AcceptedSDKKeys does NOT contain that key, the key is absent from the returned AcceptedSet. -func TestBuildAcceptedSet_TrustTheArray(t *testing.T) { - // Simulate an old-relay payload where ExpiringSDKKey is populated from sdkKey.expiring, - // but AcceptedSDKKeys only has the anchor (no expiring key in the array). - params := EnvironmentParams{ - EnvID: "env-abc", - SDKKey: "sdk-anchor", - MobileKey: "mob-primary", - ExpiringSDKKey: ExpiringSDKKey{ // legacy field — must NOT be consulted - Key: "sdk-legacy-expiring", - Expiration: expiry1, - }, - AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, - AcceptedMobileKeys: []AcceptedMobileKey{{Value: "mob-primary"}}, - } - - set, _, err := BuildAcceptedSet(params) - - require.NoError(t, err) - expected := mustBuild(t, credential.NewAcceptedSetBuilder(). - WithEnvironmentID("env-abc"). - WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). - WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) - assert.Equal(t, expected, set, "legacy sdkKey.expiring slot must not appear in AcceptedSet") -} diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index bb62f6ee..be0d3c67 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -427,39 +427,17 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { c.registerCredentialMappings(newCredential) - // A new SDK key means: - // 1. we should start a new SDK client*, but only for the anchor: there is a single upstream - // connection per environment, owned by the anchor key. Non-anchor server keys get their - // credential mappings registered above, but no upstream client — matching today's mobile-key behavior. - // 2. we should tell all event forwarding components that use an SDK key to use the new one, - // again only when it is the anchor, since events collapse to the anchor per kind. - // A new mobile key does not require starting a new SDK client, but does requiring updating any event forwarding - // components that use a mobile key. - // *Note: we only start a new SDK client in online mode. This is somewhat of an architectural hack because EnvContextImpl - // is used for both offline and online mode, yet starting up an SDK client is only relevant in online mode. This is - // because in offline mode, we already have the data (from a file) - there's no need to open a new streaming connection. - // So, the effect in offline mode when adding/removing credentials is just setting up the new credential mappings. - switch key := newCredential.(type) { - case config.SDKKey: - if key == c.keyRotator.AnchorKey() { - if !c.offline { - go c.startSDKClient(key, nil, false, c.anchorClientGen) - } - if c.metricsEventPub != nil { // metrics event publisher always uses SDK key - c.metricsEventPub.ReplaceCredential(key) - } - if c.eventDispatcher != nil { - c.eventDispatcher.ReplaceCredential(key) - } - } - case config.MobileKey: - // Mobile-key event forwarding collapses to the primary mobile key, mirroring the anchor-only - // behavior for SDK keys above: only the primary mobile key repoints the event dispatcher, so a - // non-primary mobile key accepted in the same reconcile does not steal event forwarding. - if key == c.keyRotator.MobileKey() { - if c.eventDispatcher != nil { - c.eventDispatcher.ReplaceCredential(key) - } + // Registering the credential mappings above is all that most keys require. The one extra step is + // event forwarding for mobile keys: mobile-key event forwarding collapses to the primary mobile key, + // so a newly added mobile key repoints the event dispatcher only when it is the primary mobile key + // (a non-primary mobile key accepted in the same reconcile does not steal event forwarding). + // The upstream client lifecycle and SDK-key event repointing are deliberately not handled here: there + // is a single upstream connection per environment owned by the anchor key, and it is set up exclusively + // by construction (NewEnvContext) and moved by the re-anchor sequence (commitReanchor), which also + // repoints the SDK-key event forwarders since events collapse to the anchor per kind. + if mobileKey, ok := newCredential.(config.MobileKey); ok && mobileKey == c.keyRotator.MobileKey() { + if c.eventDispatcher != nil { + c.eventDispatcher.ReplaceCredential(mobileKey) } } } @@ -469,8 +447,8 @@ func (c *envContextImpl) removeCredential(oldCredential credential.SDKCredential defer c.mu.Unlock() c.connectionMapper.RemoveConnectionMapping(sdkauth.NewScoped(c.filterKey, oldCredential)) c.envStreams.RemoveCredential(oldCredential) - // See the comment in addCredential for more context. In offline mode, there's no need to close the SDK client - // because our data comes from a file, not a streaming connection. + // In offline mode, there's no need to close the SDK client because our data comes from a file, + // not a streaming connection. if !c.offline { if sdkKey, ok := oldCredential.(config.SDKKey); ok { // The SDK client instance is tied to the SDK key, so get rid of it diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index f8ea7638..c59c4a1a 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -11,7 +11,6 @@ const ( logMsgAutoConfUpdateUnknownEnv = "Got auto-configuration update for environment %q but did not have previous configuration - will add" logMsgAutoConfDeleteUnknownEnv = "Got auto-configuration delete message for environment %s but did not have previous configuration - ignoring" logMsgAutoConfReceivedAllEnvironments = "Finished processing auto-configuration data" - logMsgKeyExpiryUnknownEnv = "Got auto-configuration key expiry message for environment %s but did not have previous configuration - ignoring" ) // relayAutoConfigActions is an implementation of the autoconfig.MessageHandler interface. The low-level @@ -33,7 +32,7 @@ func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentPar return } - set, _, buildErr := envfactory.BuildAcceptedSet(params) + set, buildErr := envfactory.BuildAcceptedSet(params) if buildErr != nil { a.r.loggers.Errorf(logMsgAutoConfEnvInitError, params.Identifiers.GetDisplayName(), buildErr) return @@ -52,7 +51,7 @@ func (a *relayAutoConfigActions) UpdateEnvironment(params envfactory.Environment env.SetTTL(params.TTL) env.SetSecureMode(params.SecureMode) - set, _, buildErr := envfactory.BuildAcceptedSet(params) + set, buildErr := envfactory.BuildAcceptedSet(params) if buildErr != nil { // Credential payloads are validated at the stream parse boundary (see StreamManager) before // being dispatched here, so a malformed set should not reach this point. Log defensively and diff --git a/relay/autoconfig_actions_test.go b/relay/autoconfig_actions_test.go index dc2c7fbb..8ca7301f 100644 --- a/relay/autoconfig_actions_test.go +++ b/relay/autoconfig_actions_test.go @@ -256,7 +256,7 @@ func TestAutoConfigAddEnvironmentWithExpiringSDKKeyDoesNotPanicWhenInitFails(t * initialEvent := makeAutoConfPutEvent() autoConfTest(t, testAutoConfDefaultConfig, &initialEvent, func(p autoConfTestParams) { params := envWithKeys.params() - require.True(t, params.ExpiringSDKKey.Defined(), + require.Len(t, params.AcceptedSDKKeys, 2, "precondition: params must carry an expiring SDK key to reach the credential-update branch") // Closing the Relay makes the next addEnvironment return (nil, nil, errAlreadyClosed). diff --git a/relay/filedata_actions.go b/relay/filedata_actions.go index 24486e8f..0ce77763 100644 --- a/relay/filedata_actions.go +++ b/relay/filedata_actions.go @@ -58,7 +58,7 @@ func (a *relayFileDataActions) AddEnvironment(ae filedata.ArchiveEnvironment) { return } - set, _, buildErr := envfactory.BuildAcceptedSet(ae.Params) + set, buildErr := envfactory.BuildAcceptedSet(ae.Params) if buildErr != nil { var malformed *credential.MalformedCredentialSetError if errors.As(buildErr, &malformed) { @@ -101,7 +101,7 @@ func (a *relayFileDataActions) UpdateEnvironment(ae filedata.ArchiveEnvironment) env.SetTTL(ae.Params.TTL) env.SetSecureMode(ae.Params.SecureMode) - set, _, buildErr := envfactory.BuildAcceptedSet(ae.Params) + set, buildErr := envfactory.BuildAcceptedSet(ae.Params) if buildErr != nil { var malformed *credential.MalformedCredentialSetError if errors.As(buildErr, &malformed) { diff --git a/relay/filedata_actions_test.go b/relay/filedata_actions_test.go index ace1f7de..dc5cb3bb 100644 --- a/relay/filedata_actions_test.go +++ b/relay/filedata_actions_test.go @@ -229,8 +229,8 @@ func TestOfflineModeDeprecatedSDKKeyIsRespectedIfExpiryInFuture(t *testing.T) { env := p.awaitEnvironment(testFileDataEnv1.Params.EnvID) // Expiring key is in the accepted set (and thus GetCredentials) until it expires. - assert.ElementsMatch(t, []credential.SDKCredential{envData.Params.SDKKey, envData.Params.ExpiringSDKKey.Key, envData.Params.EnvID}, env.GetCredentials()) - assert.ElementsMatch(t, []credential.SDKCredential{envData.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{envData.Params.SDKKey, envData.Params.AcceptedSDKKeys[1].Value, envData.Params.EnvID}, env.GetCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{envData.Params.AcceptedSDKKeys[1].Value}, env.GetDeprecatedCredentials()) }) } @@ -252,8 +252,8 @@ func TestOfflineModePrimarySDKKeyIsDeprecated(t *testing.T) { p.updateHandler.UpdateEnvironment(update2) // Both the new anchor and the expiring old key are accepted until key1 expires. - assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.SDKKey, update2.Params.ExpiringSDKKey.Key, update1.Params.EnvID}, env.GetCredentials()) - assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.SDKKey, update2.Params.AcceptedSDKKeys[1].Value, update1.Params.EnvID}, env.GetCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{update2.Params.AcceptedSDKKeys[1].Value}, env.GetDeprecatedCredentials()) update3 := RotateSDKKey("key3") p.updateHandler.UpdateEnvironment(update3) @@ -293,8 +293,8 @@ func TestOfflineModeSDKKeyCanExpire(t *testing.T) { // we'll still need to sleep at least the cleanup interval to ensure the key is expired. env := p.awaitEnvironmentFor(update1.Params.EnvID, time.Second) // Both the primary and the expiring key are in the accepted set until the expiry fires. - assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.ExpiringSDKKey.Key, update1.Params.EnvID}, env.GetCredentials()) - assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.ExpiringSDKKey.Key}, env.GetDeprecatedCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.SDKKey, update1.Params.AcceptedSDKKeys[1].Value, update1.Params.EnvID}, env.GetCredentials()) + assert.ElementsMatch(t, []credential.SDKCredential{update1.Params.AcceptedSDKKeys[1].Value}, env.GetDeprecatedCredentials()) assert.Eventually(t, func() bool { return len(env.GetDeprecatedCredentials()) == 0 diff --git a/relay/filedata_testdata_test.go b/relay/filedata_testdata_test.go index 7101e20e..7b7051c6 100644 --- a/relay/filedata_testdata_test.go +++ b/relay/filedata_testdata_test.go @@ -74,12 +74,8 @@ func RotateSDKKeyWithGracePeriod(primary config.SDKKey, expiring config.SDKKey, } return filedata.ArchiveEnvironment{ Params: envfactory.EnvironmentParams{ - EnvID: "env1", - SDKKey: primary, - ExpiringSDKKey: envfactory.ExpiringSDKKey{ - Key: expiring, - Expiration: expiry, - }, + EnvID: "env1", + SDKKey: primary, AcceptedSDKKeys: acceptedKeys, Identifiers: relayenv.EnvIdentifiers{ ProjName: "Project", From 465caad496c1c635cc3f0afb3b0af49b2b34bd35 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 21 Jul 2026 11:01:23 -0700 Subject: [PATCH 45/66] fix(concurrent-keys): align key expiry with the cleanup ticker and stop offline re-anchor from stranding the initial client (#763) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns credential reconcile with StepTime’s strictly-after expiry rule: SDK and mobile keys in reconcileSDKKeys / reconcileMobileKeys are treated as absent only when now is after the expiry instant, not when now equals it—so a reconcile at the boundary no longer drops a key the cleanup ticker would still accept for one more instant. --- internal/credential/rotator.go | 4 +- internal/credential/rotator_test.go | 26 +++++++ internal/relayenv/env_context_impl.go | 9 ++- .../env_context_reanchor_synchronous_test.go | 67 ++++++++++++++++++- 4 files changed, 102 insertions(+), 4 deletions(-) diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 2b202bd2..b9bab760 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -407,7 +407,7 @@ func reconcileAcceptedKeys[K reconcilableKey]( func (r *Rotator) reconcileSDKKeys(set AcceptedSet, now time.Time) { desired := make(map[config.SDKKey]AcceptedKey, len(set.sdkKeys)) for key, info := range set.sdkKeys { - if info.Expiry != nil && !now.Before(*info.Expiry) { + if info.Expiry != nil && now.After(*info.Expiry) { continue // already expired; treat as absent } desired[key] = info @@ -422,7 +422,7 @@ func (r *Rotator) reconcileSDKKeys(set AcceptedSet, now time.Time) { func (r *Rotator) reconcileMobileKeys(set AcceptedSet, now time.Time) { desired := make(map[config.MobileKey]AcceptedKey, len(set.mobileKeys)) for key, info := range set.mobileKeys { - if info.Expiry != nil && !now.Before(*info.Expiry) { + if info.Expiry != nil && now.After(*info.Expiry) { continue // already expired; treat as absent } desired[key] = info diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 5c4c87a3..3962a690 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -242,6 +242,32 @@ func TestReconcileAlreadyExpiredKeyIsIgnoredOnAdd(t *testing.T) { assert.ElementsMatch(t, []SDKCredential{anchor}, r.AllCredentials()) } +func TestReconcileExpiryBoundaryIsStrictlyAfter(t *testing.T) { + // The reconcile-side filter that treats an already-expired key as absent must honor the same + // strictly-after contract as StepTime (see the doc comment on StepTime): a key whose expiry lands + // exactly on `now` is still accepted by Reconcile, and only becomes absent once `now` is one instant + // past the expiry. + anchor := config.SDKKey("anchor") + staleKey := config.SDKKey("stale") + expiry := time.Unix(2000, 0) + + atBoundary := newTestRotator() + atBoundary.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: staleKey, Expiry: util.PtrOrNil(expiry)})), + expiry) + assert.Contains(t, atBoundary.AllCredentials(), SDKCredential(staleKey), "a key expiring exactly at now is still accepted by Reconcile") + + pastBoundary := newTestRotator() + pastBoundary.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: staleKey, Expiry: util.PtrOrNil(expiry)})), + expiry.Add(1*time.Millisecond)) + assert.NotContains(t, pastBoundary.AllCredentials(), SDKCredential(staleKey), "a key one instant past its expiry is treated as absent by Reconcile") +} + func TestReconcileDeExpiryRestoresKey(t *testing.T) { // When a key was accepted with a future expiry and a subsequent reconcile removes that expiry // (de-expiry), the key becomes permanent: the cleanup ticker will no longer drop it, and it is diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index be0d3c67..ce23392f 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -759,7 +759,14 @@ func (c *envContextImpl) commitReanchor(newAnchor, previousAnchor config.SDKKey, c.keyRotator.CommitAnchor(newAnchor) // A new anchor client is now authoritative, so any startSDKClient build still in flight from before // this commit is stale: bump the generation so it discards itself instead of clobbering this client. - c.anchorClientGen++ + // Only do this online: an offline commit (see reanchor above) installs no replacement client, so + // there is nothing for the bump to protect. Bumping anyway would strand the env's initial client + // build (launched with generation 0 at construction) if it is still in flight when this offline + // re-anchor commits: startSDKClient would see its generation superseded and discard the build, + // leaving GetClient() nil forever with no other build ever attempted. + if !c.offline { + c.anchorClientGen++ + } // The anchor now points at a healthy client (freshly built and Initialized, or a reused live // client), so clear any init error a prior client left behind — otherwise GetInitError() and the // request middleware would keep reporting a still-serving env as failed. diff --git a/internal/relayenv/env_context_reanchor_synchronous_test.go b/internal/relayenv/env_context_reanchor_synchronous_test.go index dee1a0bf..9483100d 100644 --- a/internal/relayenv/env_context_reanchor_synchronous_test.go +++ b/internal/relayenv/env_context_reanchor_synchronous_test.go @@ -479,11 +479,16 @@ func TestReanchorSync_Offline_CommitsWithoutBuildingClient(t *testing.T) { initialClient := requireClientReady(t, clientCh) require.Eventually(t, func() bool { return env.GetClient() == initialClient }, time.Second, 10*time.Millisecond) + envImpl := env.(*envContextImpl) + envImpl.mu.RLock() + genBefore := envImpl.anchorClientGen + envImpl.mu.RUnlock() + now := time.Unix(2000, 0) reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) // The anchor commits, but the offline branch builds no new client. - assert.Equal(t, reanchorSyncTestKey2, env.(*envContextImpl).keyRotator.AnchorKey(), "offline re-anchor commits the anchor") + assert.Equal(t, reanchorSyncTestKey2, envImpl.keyRotator.AnchorKey(), "offline re-anchor commits the anchor") select { case c := <-clientCh: t.Fatalf("an offline re-anchor must not build a new SDK client, got: %v", c.Key) @@ -491,6 +496,15 @@ func TestReanchorSync_Offline_CommitsWithoutBuildingClient(t *testing.T) { } assert.NoError(t, env.GetInitError()) + // The generation guard exists to protect a replacement client's install from a stale, still-in-flight + // build. An offline commit installs no replacement, so bumping it protects nothing -- it would only + // strand a build launched before this commit (e.g. the initial client at construction, generation 0) + // by making it see itself as superseded when it later finishes. Offline commits leave it untouched. + envImpl.mu.RLock() + genAfter := envImpl.anchorClientGen + envImpl.mu.RUnlock() + assert.Equal(t, genBefore, genAfter, "an offline re-anchor commit must not advance anchorClientGen") + // The single offline client survives the rotation: it is not closed and GetClient still finds it. if !helpers.AssertChannelNotClosed(t, initialClient.CloseCh, 100*time.Millisecond, "the offline env's only client must not be closed by a re-anchor") { @@ -499,6 +513,57 @@ func TestReanchorSync_Offline_CommitsWithoutBuildingClient(t *testing.T) { assert.Same(t, initialClient, env.GetClient(), "GetClient keeps returning the offline client after re-anchor") } +// TestReanchorSync_Offline_ReanchorDuringInitialBuildDoesNotStrandClient drives the failure scenario +// the anchorClientGen guard above prevents: an offline re-anchor commits while the environment's +// initial client build (launched at construction with generation 0) is still in flight. Before the +// fix, that commit's unconditional generation bump made the in-flight build see itself as superseded +// once it finished, so it discarded itself -- and because an offline re-anchor never builds a +// replacement, the environment was left with GetClient() permanently nil. With the fix, the offline +// commit leaves the generation untouched, so the initial build is not superseded and installs +// normally once it completes. +func TestReanchorSync_Offline_ReanchorDuringInitialBuildDoesNotStrandClient(t *testing.T) { + envConfig := st.EnvMain.Config + envConfig.Offline = true + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + healthy := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + gate := make(chan struct{}) + entered := make(chan struct{}, 1) + factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == envConfig.SDKKey { + // The initial anchor build: block until the offline re-anchor below has committed. + entered <- struct{}{} + <-gate + } + return healthy(sdkKey, cfg, timeout) + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) + defer env.Close() + + envImpl := env.(*envContextImpl) + <-entered // the initial build is blocked; no client is installed yet. + + // While that build is in flight, an offline re-anchor commits to a new anchor. The offline branch + // builds no replacement client, so nothing is installed for the new anchor either. + now := time.Unix(2000, 0) + reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) + require.Equal(t, reanchorSyncTestKey2, envImpl.keyRotator.AnchorKey(), "offline re-anchor commits the anchor") + assert.Nil(t, env.GetClient(), "no client is installed yet: the initial build is still blocked and the offline commit built none") + + // Release the initial build. It must not see itself as superseded (the offline commit above did not + // advance anchorClientGen), so it installs normally and GetClient starts returning it. + close(gate) + initialClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == initialClient }, time.Second, 10*time.Millisecond) + assert.NoError(t, env.GetInitError()) +} + // TestReanchorSync_RollbackWithImmediateRevocationKeepsOldAnchorServing covers the edge where a // reconcile both moves the anchor to a new key AND immediately revokes the current anchor (no grace // expiry), and the new anchor's client fails to build. The re-anchor rolls back, backing out just the From 163e410aeddcf058ad3a7b5df7a6e0853c5c15ab Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 21 Jul 2026 11:24:10 -0700 Subject: [PATCH 46/66] fix(concurrent-keys): reject mobile-key payloads that arrive without a primary mobile key (#761) BuildAcceptedSet now treats a non-empty mobileKeys[] with an undefined mobKey as malformed (via NewPrimaryMobileKeyMissingError), matching the existing SDK anchor rules. Accepting that shape would clear the primary on reconcile while event forwarding could keep using the old primary. --- .../autoconfig/stream_manager_errors_test.go | 112 +++++++++++------- internal/credential/accepted_set.go | 14 ++- internal/envfactory/reconcile_helper.go | 14 ++- internal/envfactory/reconcile_helper_test.go | 43 +++++++ 4 files changed, 138 insertions(+), 45 deletions(-) diff --git a/internal/autoconfig/stream_manager_errors_test.go b/internal/autoconfig/stream_manager_errors_test.go index 49353725..9ea923a0 100644 --- a/internal/autoconfig/stream_manager_errors_test.go +++ b/internal/autoconfig/stream_manager_errors_test.go @@ -34,55 +34,81 @@ func eventShouldCauseStreamRestart(t *testing.T, event httphelpers.SSEEvent) { } // A credential payload that is valid JSON and a structurally valid event, but whose credential set -// cannot be built (here: an undefined anchor SDK key), must be caught at the parse boundary: the -// previous state is preserved (no AddEnvironment/UpdateEnvironment dispatched) and the stream is -// restarted so the backend resends a fresh put (design §9). This is verified for both patch and put, -// since both paths run the validation before the version is recorded. +// cannot be built (e.g. an undefined anchor SDK key, or mobile keys with no designated primary), must +// be caught at the parse boundary: the previous state is preserved (no AddEnvironment/UpdateEnvironment +// dispatched) and the stream is restarted so the backend resends a fresh put (design §9). This is +// verified for both patch and put, since both paths run the validation before the version is recorded. func TestMalformedCredentialPayloadCausesStreamRestart(t *testing.T) { - malformedEnv := testEnv1 - malformedEnv.SDKKey = envfactory.SDKKeyRep{Value: config.SDKKey("")} // undefined anchor + // Each shape is a distinct way BuildAcceptedSet rejects a structurally malformed payload; all ride + // the same malformed-payload machinery. + malformedShapes := []struct { + name string + make func(envfactory.EnvironmentRep) envfactory.EnvironmentRep + }{ + { + name: "undefined anchor SDK key", + make: func(env envfactory.EnvironmentRep) envfactory.EnvironmentRep { + env.SDKKey = envfactory.SDKKeyRep{Value: config.SDKKey("")} + return env + }, + }, + { + name: "mobile keys without a designated primary", + make: func(env envfactory.EnvironmentRep) envfactory.EnvironmentRep { + env.MobKey = config.MobileKey("") // no primary designated... + env.MobileKeys = []envfactory.ConcurrentKeyRep{{Key: "mob-1", Value: "mobkey1"}} // ...but non-empty + return env + }, + }, + } - t.Run("patch", func(t *testing.T) { - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - <-p.requestsCh - p.stream.Enqueue(makePatchEnvEvent(malformedEnv)) - select { - case m := <-p.messageHandler.received: - require.Failf(t, "unexpected message", - "must not dispatch for a malformed payload, got %s", m) - case <-p.requestsCh: // reconnect request == stream restart - p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") - case <-time.After(time.Second): - require.Fail(t, "timed out waiting for stream restart") - } - }) - }) + for _, shape := range malformedShapes { + t.Run(shape.name, func(t *testing.T) { + malformedEnv := shape.make(testEnv1) - t.Run("put", func(t *testing.T) { - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - <-p.requestsCh - p.stream.Enqueue(makeEnvPutEvent(malformedEnv)) - // The malformed env is skipped (no add/update); a put still reports ReceivedAllEnvironments, - // which we tolerate. We require that the stream restarts and that no add/update is dispatched. - deadline := time.After(2 * time.Second) - for { - select { - case m := <-p.messageHandler.received: - if m.add != nil || m.update != nil { + t.Run("patch", func(t *testing.T) { + streamManagerTest(t, nil, func(p streamManagerTestParams) { + p.startStream() + <-p.requestsCh + p.stream.Enqueue(makePatchEnvEvent(malformedEnv)) + select { + case m := <-p.messageHandler.received: require.Failf(t, "unexpected message", - "must not dispatch add/update for a malformed payload, got %s", m) + "must not dispatch for a malformed payload, got %s", m) + case <-p.requestsCh: // reconnect request == stream restart + p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") + case <-time.After(time.Second): + require.Fail(t, "timed out waiting for stream restart") } - case <-p.requestsCh: // reconnect request == stream restart - p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") - return - case <-deadline: - require.Fail(t, "timed out waiting for stream restart") - } - } + }) + }) + + t.Run("put", func(t *testing.T) { + streamManagerTest(t, nil, func(p streamManagerTestParams) { + p.startStream() + <-p.requestsCh + p.stream.Enqueue(makeEnvPutEvent(malformedEnv)) + // The malformed env is skipped (no add/update); a put still reports ReceivedAllEnvironments, + // which we tolerate. We require that the stream restarts and that no add/update is dispatched. + deadline := time.After(2 * time.Second) + for { + select { + case m := <-p.messageHandler.received: + if m.add != nil || m.update != nil { + require.Failf(t, "unexpected message", + "must not dispatch add/update for a malformed payload, got %s", m) + } + case <-p.requestsCh: // reconnect request == stream restart + p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") + return + case <-deadline: + require.Fail(t, "timed out waiting for stream restart") + } + } + }) + }) }) - }) + } } // A put carrying malformed credential payloads must not corrupt the persistent cache: valid envs are diff --git a/internal/credential/accepted_set.go b/internal/credential/accepted_set.go index dc65c3c6..24dee4f3 100644 --- a/internal/credential/accepted_set.go +++ b/internal/credential/accepted_set.go @@ -60,7 +60,10 @@ var errAcceptedSetMissingSDKKey = errors.New("accepted credential set must conta // sdkKeys[] — a violation of the invariant that the designated anchor is one of the accepted keys. // 2. The primary mobile key (mobKey) is defined but not present in mobileKeys[] — the mobile-key // analogue of the anchor invariant. -// 3. An entry in sdkKeys[] or mobileKeys[] has an empty value — a credential that would be +// 3. mobileKeys[] is non-empty but no primary mobile key (mobKey) is designated — accepting it would +// clear the environment's primary mobile key on reconcile with no repoint, so event forwarding +// would keep using the previous (possibly revoked) primary. (No mobile keys at all is valid.) +// 4. An entry in sdkKeys[] or mobileKeys[] has an empty value — a credential that would be // accepted by relay but can never authenticate any SDK. // // Validation happens before Reconcile is called; Rotator.Reconcile trusts the set it is handed. @@ -98,6 +101,15 @@ func NewPrimaryMobileKeyNotInSetError() *MalformedCredentialSetError { return &MalformedCredentialSetError{msg: "malformed credential set: primary mobile key is not present in mobileKeys[]"} } +// NewPrimaryMobileKeyMissingError returns a MalformedCredentialSetError for a payload that carries a +// non-empty mobileKeys[] array but leaves the primary mobile key (mobKey) undefined — no default is +// designated. Accepting it would clear the environment's primary mobile key on reconcile with no +// repoint, so event forwarding would keep using the previous (possibly revoked) primary. There are no +// secrets to omit from the message. +func NewPrimaryMobileKeyMissingError() *MalformedCredentialSetError { + return &MalformedCredentialSetError{msg: "malformed credential set: mobileKeys[] is non-empty but no primary mobile key is designated"} +} + // NewEmptyCredentialError returns a MalformedCredentialSetError for a key-array entry whose // value field is empty. kind is "sdkKeys" or "mobileKeys"; key is the entry's wire "key" identifier // (may be empty for old-format payloads that synthesize from the singular fields). diff --git a/internal/envfactory/reconcile_helper.go b/internal/envfactory/reconcile_helper.go index 28f8d426..4a5f418e 100644 --- a/internal/envfactory/reconcile_helper.go +++ b/internal/envfactory/reconcile_helper.go @@ -22,7 +22,9 @@ import ( // // A *credential.MalformedCredentialSetError is returned (with an empty AcceptedSet) for a // structurally malformed payload: an undefined anchor (params.SDKKey not set), a defined anchor that -// is absent from params.AcceptedSDKKeys, or an array entry with an empty value. The caller must +// is absent from params.AcceptedSDKKeys, a defined primary mobile key (params.MobileKey) that is +// absent from params.AcceptedMobileKeys, a non-empty params.AcceptedMobileKeys with no designated +// primary (params.MobileKey undefined), or an array entry with an empty value. The caller must // preserve the previous accepted state and, for RAC handlers, reconnect the stream with jitter to // force a fresh put. This is the single home for the anchor invariant. func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, error) { @@ -80,6 +82,16 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, error) return credential.AcceptedSet{}, credential.NewPrimaryMobileKeyNotInSetError() } + // A non-empty mobileKeys[] with no designated primary (undefined mobKey) is malformed: the reconcile + // would clear the rotator's primary mobile key without a repoint, so event forwarding would keep + // using the previous (possibly revoked) primary — silent misattribution rather than a loud + // rejection. (No mobile keys at all — empty array and undefined mobKey — stays valid: a + // server-side-only environment. Old-format payloads synthesize the array from mobKey only, so an + // undefined mobKey yields an empty array and is unaffected.) + if len(params.AcceptedMobileKeys) > 0 && !params.MobileKey.Defined() { + return credential.AcceptedSet{}, credential.NewPrimaryMobileKeyMissingError() + } + set, err := b.Build() if err != nil { return credential.AcceptedSet{}, err diff --git a/internal/envfactory/reconcile_helper_test.go b/internal/envfactory/reconcile_helper_test.go index a5b63cad..df87d178 100644 --- a/internal/envfactory/reconcile_helper_test.go +++ b/internal/envfactory/reconcile_helper_test.go @@ -220,6 +220,49 @@ func TestBuildAcceptedSet_NoMobileKey(t *testing.T) { assert.Equal(t, expected, set) } +// TestBuildAcceptedSet_MobileKeysWithoutPrimary verifies the complement of the primary-mobile-in-array +// invariant: a non-empty mobileKeys[] with no designated primary (undefined mobKey) is rejected as a +// *credential.MalformedCredentialSetError. Without this guard the reconcile would clear the rotator's +// primary mobile key with no repoint, silently forwarding events under the previous (possibly revoked) +// primary instead of loudly rejecting the payload. +func TestBuildAcceptedSet_MobileKeysWithoutPrimary(t *testing.T) { + params := EnvironmentParams{ + EnvID: "env-abc", + SDKKey: "sdk-anchor", + MobileKey: "", // no primary designated... + AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, + AcceptedMobileKeys: []AcceptedMobileKey{ + {Key: "mob-1", Value: "mob-primary"}, // ...but the array is non-empty + }, + } + _, err := BuildAcceptedSet(params) + + require.Error(t, err) + var malformed *credential.MalformedCredentialSetError + require.True(t, errors.As(err, &malformed)) + assert.Contains(t, malformed.Error(), "no primary mobile key is designated") +} + +// TestBuildAcceptedSet_EmptyMobileArrayValid verifies the boundary of the guard above: an empty +// mobileKeys[] with an undefined mobKey (a server-side-only environment) is valid — the guard fires +// only when the array is non-empty, so no primary mobile key is designated and nothing is rejected. +func TestBuildAcceptedSet_EmptyMobileArrayValid(t *testing.T) { + params := EnvironmentParams{ + EnvID: "env-abc", + SDKKey: "sdk-anchor", + MobileKey: "", // undefined + AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, + AcceptedMobileKeys: []AcceptedMobileKey{}, // empty + } + set, err := BuildAcceptedSet(params) + + require.NoError(t, err) + expected := mustBuild(t, credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")})) + assert.Equal(t, expected, set) +} + // TestBuildAcceptedSet_AnchorUndefined verifies that an undefined anchor (empty SDKKey) yields a // *credential.MalformedCredentialSetError: no anchor was designated, so Build rejects the set. func TestBuildAcceptedSet_AnchorUndefined(t *testing.T) { From 4b19b77f3e3a65babeb0e3a7f006b03813631257 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 22 Jul 2026 12:51:39 -0700 Subject: [PATCH 47/66] fix(deps): bump grpc, x/net, x/text to patch disclosed CVEs (#772) - Bump `google.golang.org/grpc` v1.80.0 -> v1.82.1 (fixes HIGH severity [GHSA-hrxh-6v49-42gf](https://github.com/advisories/GHSA-hrxh-6v49-42gf)) - Bump `golang.org/x/net` v0.55.0 -> v0.56.0 (fixes CVE-2026-46600) - Bump `golang.org/x/text` v0.37.0 -> v0.39.0 (fixes CVE-2026-56852) --- go.mod | 24 ++++++++++------------ go.sum | 65 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 44 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index 72681936..b9e8da7f 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/prometheus/client_golang v1.23.2 // indirect; override to address CVE-2022-21698 github.com/stretchr/testify v1.11.1 go.opencensus.io v0.24.0 - golang.org/x/sync v0.20.0 + golang.org/x/sync v0.21.0 gopkg.in/gcfg.v1 v1.2.3 ) @@ -62,7 +62,7 @@ require ( github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b github.com/klauspost/compress v1.18.5 github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e - golang.org/x/crypto v0.52.0 + golang.org/x/crypto v0.53.0 ) require ( @@ -79,8 +79,6 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect - github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-kit/log v0.2.1 // indirect @@ -125,20 +123,20 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect - go.opentelemetry.io/otel v1.42.0 // indirect - go.opentelemetry.io/otel/metric v1.42.0 // indirect - go.opentelemetry.io/otel/trace v1.42.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/net v0.55.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.39.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/api v0.272.0 // indirect google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c // indirect - google.golang.org/grpc v1.80.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/DataDog/dd-trace-go.v1 v1.61.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index c36e8c0e..4700023b 100644 --- a/go.sum +++ b/go.sum @@ -118,8 +118,8 @@ github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6D github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= -github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -131,6 +131,7 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= @@ -519,16 +520,16 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.6 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= -go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= -go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= -go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= -go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= -go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= -go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= -go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= -go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= -go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= -go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= @@ -540,8 +541,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -574,8 +575,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -611,8 +612,8 @@ golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -634,8 +635,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -690,8 +691,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -701,8 +702,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -749,8 +750,8 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -812,10 +813,10 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d h1:vsOm753cOAMkt76efriTCDKjpCbK18XGHMJHo0JUKhc= google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:0oz9d7g9QLSdv9/lgbIjowW1JoxMbxmBVNe8i6tORJI= -google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 h1:41r6JMbpzBMen0R/4TZeeAmGXSJC7DftGINUodzTkPI= -google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c h1:xgCzyF2LFIO/0X2UAoVRiXKU5Xg6VjToG4i2/ecSswk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -829,8 +830,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 7efb32dd47093a0c5914d4ac0aaae06dbb42319e Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Thu, 23 Jul 2026 10:50:47 -0700 Subject: [PATCH 48/66] fix(deps): bump eventsource to v1.11.1 (#770) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dependency-only change: github.com/launchdarkly/eventsource is updated v1.11.0 → v1.11.1 in go.mod and go.sum; no Relay source changes. v1.11.1 fixes a stream-shutdown deadlock when Close races with an in-flight event: the stream could stop closing Events, so shutdown paths that drain the stream (e.g. auto-config StreamManager / consumeStream) could block indefinitely and hang relay.Close(). --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b9e8da7f..8cf054e4 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/kardianos/minwinsvc v1.0.2 - github.com/launchdarkly/eventsource v1.11.0 + github.com/launchdarkly/eventsource v1.11.1 github.com/launchdarkly/go-configtypes v1.2.2 github.com/launchdarkly/go-jsonstream/v3 v3.1.1 github.com/launchdarkly/go-sdk-common/v3 v3.5.0 diff --git a/go.sum b/go.sum index 4700023b..476a5716 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,8 @@ github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e/go.mod h1:cQRkOAs0LGcfIs6RSsHNqwhzItUZooyhpqPv0hgiQZM= github.com/launchdarkly/ccache v1.1.0 h1:voD1M+ZJXR3MREOKtBwgTF9hYHl1jg+vFKS/+VAkR2k= github.com/launchdarkly/ccache v1.1.0/go.mod h1:TlxzrlnzvYeXiLHmesMuvoZetu4Z97cV1SsdqqBJi1Q= -github.com/launchdarkly/eventsource v1.11.0 h1:aAdvh2XmtXA17QsRFL0XKHURMqhxg7J+CceQmhSzBas= -github.com/launchdarkly/eventsource v1.11.0/go.mod h1:dU+rZxkPOlGPsyJPpiDqiepAcFwIITDUClY9+A6RrMw= +github.com/launchdarkly/eventsource v1.11.1 h1:R0g6QbfuIByA43wkcKswOg5Ku6pgIW0TJEGOqLTP6vE= +github.com/launchdarkly/eventsource v1.11.1/go.mod h1:dU+rZxkPOlGPsyJPpiDqiepAcFwIITDUClY9+A6RrMw= github.com/launchdarkly/go-configtypes v1.2.2 h1:IXfC7puQpUSctkNfd0L3t6kvlVIQwszBDWpnoiYxk8g= github.com/launchdarkly/go-configtypes v1.2.2/go.mod h1:KAwNI0N8ZuAZecfBga9sAv/LwlChEA1RDJ6+pxbxwhk= github.com/launchdarkly/go-jsonstream/v3 v3.1.1 h1:ugupp2eNtwVbr69KCdeUrm1vUf1/3ju4Wdliaob95uY= From 9b2d3a1f5ca637af99af108f1448798fa93d0b7d Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Fri, 24 Jul 2026 10:18:09 -0700 Subject: [PATCH 49/66] test(concurrent-keys): even more integration tests (#756) --- .../reload_restart_redis_test.go | 161 ++++++++++++++++++ relay/concurrent_keys_auth_test.go | 55 ++++++ 2 files changed, 216 insertions(+) create mode 100644 internal/autoconfigcache/reload_restart_redis_test.go diff --git a/internal/autoconfigcache/reload_restart_redis_test.go b/internal/autoconfigcache/reload_restart_redis_test.go new file mode 100644 index 00000000..14e509dd --- /dev/null +++ b/internal/autoconfigcache/reload_restart_redis_test.go @@ -0,0 +1,161 @@ +//go:build redis_unit_tests +// +build redis_unit_tests + +package autoconfigcache + +// Verifies that a multi-key environment written to the real Redis-backed AutoConfig cache survives a +// process restart: a fresh StreamManager, given the same Redis cache but a config stream that delivers +// nothing, reloads the environment from the cache with its sdkKeys[]/mobileKeys[] arrays intact. +// +// This is the restart-survival half of the "cache integrity across restart" scenario (SDK-2609 #10); +// the malformed-put-preserves-cache half is covered by the StreamManager unit test +// TestMalformedCredentialPayloadPreservesEnvironmentCache. It exercises the production classes +// (StreamManager + redisStore) against an actual Redis, with no test doubles for the cache itself. +// +// Requires a Redis server on localhost (the redis_unit_tests build tag; CI provides one). + +import ( + "context" + "net/url" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/autoconfig" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/httpconfig" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/configsource" + + "github.com/launchdarkly/go-configtypes" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + helpers "github.com/launchdarkly/go-test-helpers/v3" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + restartRedisURL = "redis://localhost:6379" + restartCacheKey = "sdk2609-restart-reload-test" + restartProtocolV2 = 2 +) + +// capturingHandler is a minimal autoconfig.MessageHandler that records the environments it is told to +// add, so the test can inspect what a StreamManager reloaded from the cache. +type capturingHandler struct { + added chan envfactory.EnvironmentParams +} + +func newCapturingHandler() *capturingHandler { + return &capturingHandler{added: make(chan envfactory.EnvironmentParams, 10)} +} + +func (h *capturingHandler) AddEnvironment(params envfactory.EnvironmentParams) { h.added <- params } +func (h *capturingHandler) UpdateEnvironment(params envfactory.EnvironmentParams) { h.added <- params } +func (h *capturingHandler) DeleteEnvironment(config.EnvironmentID) {} +func (h *capturingHandler) ReceivedAllEnvironments() {} +func (h *capturingHandler) AddFilter(envfactory.FilterParams) {} +func (h *capturingHandler) DeleteFilter(config.FilterID) {} + +func TestConcurrentKeysCacheReloadSurvivesRestart(t *testing.T) { + const ( + envID = config.EnvironmentID("multikey-env") + anchorSDK = config.SDKKey("sdk-anchor") + extraSDK = config.SDKKey("sdk-extra") + anchorMob = config.MobileKey("mob-anchor") + extraMob = config.MobileKey("mob-extra") + ) + + cacheConfig := func() config.Config { + cfg := config.Config{} + cfg.AutoConfig.Key = config.AutoConfigKey("test-key") + cfg.AutoConfig.CacheKey = restartCacheKey + cfg.Redis.URL, _ = configtypes.NewOptURLAbsoluteFromString(restartRedisURL) + return cfg + } + + loggers := ldlog.NewDisabledLoggers() + httpConfig, err := httpconfig.NewHTTPConfig(config.ProxyConfig{}, config.HTTPConfig{}, nil, "", loggers) + require.NoError(t, err) + + newStreamManager := func(streamURL string, store Store, handler autoconfig.MessageHandler) *autoconfig.StreamManager { + u, parseErr := url.Parse(streamURL) + require.NoError(t, parseErr) + return autoconfig.NewStreamManager(cacheConfig().AutoConfig.Key, u, handler, httpConfig, + time.Millisecond, restartProtocolV2, loggers, store) + } + + // A multi-key environment (anchor + one extra SDK key, anchor + one extra mobile key) via the array + // wire format. + rep := envfactory.EnvironmentRep{ + EnvID: envID, + EnvKey: "multikey", + EnvName: "Multi-Key Env", + ProjKey: "multikey-proj", + ProjName: "Multi-Key Project", + SDKKey: envfactory.SDKKeyRep{Value: anchorSDK}, + MobKey: anchorMob, + SDKKeys: []envfactory.ConcurrentKeyRep{ + {Key: "anchor-sdk", Value: string(anchorSDK)}, + {Key: "extra-sdk", Value: string(extraSDK)}, + }, + MobileKeys: []envfactory.ConcurrentKeyRep{ + {Key: "anchor-mob", Value: string(anchorMob)}, + {Key: "extra-mob", Value: string(extraMob)}, + }, + Version: 1, + } + + // Start from a clean cache so a previous run's entry can't stand in for this run's write. + seed, err := NewStore(cacheConfig(), loggers) + require.NoError(t, err) + require.NoError(t, seed.SetAll(context.Background(), autoconfig.PutContent{})) + require.NoError(t, seed.Close()) + + // First run: a StreamManager receives the multi-key put over its stream and persists it to Redis. + store1, err := NewStore(cacheConfig(), loggers) + require.NoError(t, err) + putEvent := configsource.MakeAutoConfigPutEvent(rep) + liveStream := configsource.NewRACMock(t, &putEvent) + sm1 := newStreamManager(liveStream.URL, store1, newCapturingHandler()) + helpers.RequireValue(t, sm1.Start(), 5*time.Second, "timed out waiting for the first stream to be ready") + + require.Eventually(t, func() bool { + content, _ := store1.GetAll(context.Background()) + if content == nil { + return false + } + _, ok := content.Environments[envID] + return ok + }, 5*time.Second, 10*time.Millisecond, "the multi-key env was not persisted to the Redis cache") + + sm1.Close() // simulate process shutdown (also closes store1's Redis client) + + // Second run ("after the restart"): a fresh StreamManager with the same Redis cache but a config + // stream that connects and delivers nothing, so the cache is the only possible source. + store2, err := NewStore(cacheConfig(), loggers) + require.NoError(t, err) + silentStream := configsource.NewRACMock(t, nil) + handler2 := newCapturingHandler() + sm2 := newStreamManager(silentStream.URL, store2, handler2) + defer sm2.Close() + helpers.RequireValue(t, sm2.Start(), 5*time.Second, "timed out waiting for the second stream to be ready") + + reloaded := helpers.RequireValue(t, handler2.added, 5*time.Second, + "the environment was not reloaded from the Redis cache after restart") + assert.Equal(t, envID, reloaded.EnvID) + + // The multi-key arrays survived the cache round-trip: both the anchor and the non-anchor key are + // present in the reloaded accepted set, for SDK and mobile keys alike. + var sdkValues []config.SDKKey + for _, k := range reloaded.AcceptedSDKKeys { + sdkValues = append(sdkValues, k.Value) + } + assert.ElementsMatch(t, []config.SDKKey{anchorSDK, extraSDK}, sdkValues) + + var mobValues []config.MobileKey + for _, k := range reloaded.AcceptedMobileKeys { + mobValues = append(mobValues, k.Value) + } + assert.ElementsMatch(t, []config.MobileKey{anchorMob, extraMob}, mobValues) +} diff --git a/relay/concurrent_keys_auth_test.go b/relay/concurrent_keys_auth_test.go index 993980f3..e3ae1dd7 100644 --- a/relay/concurrent_keys_auth_test.go +++ b/relay/concurrent_keys_auth_test.go @@ -494,6 +494,61 @@ func TestConcurrentKeysRAC_NonAnchorConnectionSurvivesAnchorRotation(t *testing. h.assertSDKEndpointsAvailability(true, extraSDKKey, "", "") } +// multiKeyArchiveEnvWithAnchor is multiKeyArchiveEnv with a caller-chosen anchor SDK key, used to +// rotate the anchor across an archive reload. The chosen anchor must also appear in sdkKeys. +func multiKeyArchiveEnvWithAnchor(anchor config.SDKKey, sdkKeys []envfactory.AcceptedSDKKey, mobileKeys []envfactory.AcceptedMobileKey) filedata.ArchiveEnvironment { + env := multiKeyArchiveEnv(sdkKeys, mobileKeys) + env.Params.SDKKey = anchor + return env +} + +// Rotating the anchor via an offline archive reload. +// +// A downstream SDK connected on a non-anchor key keeps its stream when the archive reloads with the +// anchor rotated to a brand-new key: the new anchor authenticates, the old anchor stops, and the +// non-anchor sibling is undisturbed. Offline re-anchoring reuses the environment's single file-data +// client rather than swapping an upstream connection (that swap is the RAC path, exercised by +// TestConcurrentKeysRAC_RotatingAnchorUpdatesUpstreamClient), so no new client is built and the open +// connection survives. The non-anchor expiry-via-reload half of the offline reload-rotation scenario is +// covered by TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyExpires. +func TestConcurrentKeysOffline_AnchorRotationViaArchiveReload(t *testing.T) { + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + env := p.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + // Connect a downstream SDK on the non-anchor key and confirm it is live before rotating. + req := sharedtest.BuildRequestWithAuth("GET", "/all", extraSDKKey, nil) + sharedtest.WithStreamRequest(t, req, p.relay, func(eventCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) + + // Reload the archive with the anchor rotated to a brand-new key; the non-anchor extra SDK key + // stays accepted and the mobile keys are unchanged. + p.updateHandler.UpdateEnvironment(multiKeyArchiveEnvWithAnchor( + rotatedAnchorSDKKey, + []envfactory.AcceptedSDKKey{{Value: rotatedAnchorSDKKey}, {Value: extraSDKKey}}, + defaultAcceptedMobileKeys(), + )) + + // The offline re-anchor reuses the single file-data client, so the non-anchor stream is not + // torn down by the swap. + assertStreamStaysOpen(t, eventCh, 500*time.Millisecond) + }) + + // Offline re-anchoring commits without building a replacement upstream client. + p.shouldNotCreateClient(200 * time.Millisecond) + + awaitCredentialRemoved(t, p.relay, anchorSDKKey) + + // The rotated anchor and the retained non-anchor key authenticate; the old anchor no longer does. + p.assertSDKEndpointsAvailability(true, rotatedAnchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, "", "") + p.assertSDKEndpointsAvailability(false, anchorSDKKey, "", "") + }) +} + // awaitStreamClosed reads from a WithStreamRequest event channel until the stream-closed sentinel // (a nil event) arrives, failing if the timeout elapses first. Non-nil events are ignored. func awaitStreamClosed(t *testing.T, eventCh <-chan eventsource.Event, timeout time.Duration) { From d98f5379f67c5ab9866aebc85b6013115608ee46 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Fri, 24 Jul 2026 10:19:19 -0700 Subject: [PATCH 50/66] test(concurrent-keys): close test coverage gaps throughout the feature branch (#764) --- .../autoconfig/stream_manager_errors_test.go | 91 ++++++++ internal/envfactory/env_rep_test.go | 74 ++++++ .../events/event_payload_regression_test.go | 182 +++++++++++---- .../env_context_reanchor_behavior_test.go | 187 +++++++++++++++ .../env_context_reanchor_bigsegment_test.go | 102 ++++----- .../env_context_reanchor_close_test.go | 109 +++++++++ internal/sharedtest/configsource/rac_mock.go | 22 ++ relay/autoconfig_actions_test.go | 21 +- relay/concurrent_keys_defensive_test.go | 94 ++++++++ relay/concurrent_keys_lifecycle_test.go | 212 ++++++++++++++++++ relay/concurrent_keys_reanchor_test.go | 210 +++++++++++++++++ relay/endpoints_status_test.go | 92 ++++++++ 12 files changed, 1290 insertions(+), 106 deletions(-) create mode 100644 internal/relayenv/env_context_reanchor_behavior_test.go create mode 100644 internal/relayenv/env_context_reanchor_close_test.go create mode 100644 relay/concurrent_keys_defensive_test.go create mode 100644 relay/concurrent_keys_lifecycle_test.go create mode 100644 relay/concurrent_keys_reanchor_test.go diff --git a/internal/autoconfig/stream_manager_errors_test.go b/internal/autoconfig/stream_manager_errors_test.go index 9ea923a0..dc3a4498 100644 --- a/internal/autoconfig/stream_manager_errors_test.go +++ b/internal/autoconfig/stream_manager_errors_test.go @@ -194,6 +194,97 @@ func TestMalformedCredentialPayloadPreservesEnvironmentCache(t *testing.T) { }) } +// malformedRecoveryTest drives the full malformed-payload recovery loop at the stream-manager level. +// The first connection serves a malformed credential payload (via the SequentialHandler pattern from +// errorShouldCauseReconnect); the second connection, reached after the reconnect, serves a corrected +// put. It asserts the three parts of the story: (a) the malformed payload dispatches no add/update, +// (b) the malformed payload triggers a reconnect, and (c) the corrected put's environment then +// dispatches with the corrected credential set. +// +// correctedEnv deliberately carries the same version as the malformed payload the caller passes in. +// Validation runs before the version is recorded, so the malformed version is never stored; a corrected +// put at that same version must therefore still be treated as new and dispatch, not be deduplicated away. +func malformedRecoveryTest(t *testing.T, malformedEvent httphelpers.SSEEvent, correctedEnv envfactory.EnvironmentRep) { + malformedHandler, malformedStream := httphelpers.SSEHandler(&malformedEvent) + defer malformedStream.Close() + + correctedPut := makeEnvPutEvent(correctedEnv) + correctedHandler, correctedStream := httphelpers.SSEHandler(&correctedPut) + defer correctedStream.Close() + + handler := httphelpers.SequentialHandler( + malformedHandler, // first connection serves the malformed payload + correctedHandler, // connection after the reconnect serves the corrected put + ) + + streamManagerTestWithStreamHandler(t, handler, correctedStream, noopTestCache{}, func(p streamManagerTestParams) { + p.startStream() + <-p.requestsCh // first connection + + // (a) + (b): the malformed payload must trigger a reconnect and must not dispatch any + // add/update. A malformed put still reports ReceivedAllEnvironments, which we tolerate; a + // malformed patch reports nothing. Loop until we observe the reconnect. + sawReconnect := false + deadline := time.After(2 * time.Second) + for !sawReconnect { + select { + case m := <-p.messageHandler.received: + if m.add != nil || m.update != nil { + require.Failf(t, "unexpected message", + "must not dispatch add/update for the malformed payload, got %s", m) + } + case <-p.requestsCh: // reconnect request == stream restart + sawReconnect = true + case <-deadline: + require.Fail(t, "timed out waiting for stream restart") + } + } + p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") + + // (c): after the reconnect the corrected put's environment must dispatch as an add carrying the + // corrected credential set. Because the malformed version was never recorded, the corrected put + // at the same version is not deduplicated. + msg := p.requireMessage() + require.NotNil(t, msg.add, "the corrected put must dispatch an add after recovery") + assert.Equal(t, correctedEnv.EnvID, msg.add.EnvID) + assert.Equal(t, correctedEnv.SDKKey.Value, msg.add.SDKKey, + "the corrected put's anchor key must dispatch") + assert.Equal(t, []envfactory.AcceptedSDKKey{{Value: correctedEnv.SDKKey.Value}}, msg.add.AcceptedSDKKeys, + "the corrected put's accepted set must dispatch") + }) +} + +// TestMalformedCredentialPayloadRecoversAfterReconnect extends the malformed-payload story past the +// reconnect: once the backend resends a corrected put on the new connection, the update must dispatch. +// This pins that validating at the parse boundary (before the version is recorded) leaves the receiver +// able to accept a corrected put at the same version — a validate-after-record ordering would dedup it. +func TestMalformedCredentialPayloadRecoversAfterReconnect(t *testing.T) { + // The corrected env carries the same version as every malformed variant below (testEnv1.Version). + correctedEnv := testEnv1 + require.Equal(t, 10, correctedEnv.Version, "guarding the same-version premise of this test") + + t.Run("malformed put recovers, corrected put at same version dispatches", func(t *testing.T) { + malformed := testEnv1 + malformed.SDKKey = envfactory.SDKKeyRep{Value: config.SDKKey("")} // undefined anchor + malformedRecoveryTest(t, makeEnvPutEvent(malformed), correctedEnv) + }) + + t.Run("malformed patch recovers, corrected put dispatches", func(t *testing.T) { + malformed := testEnv1 + malformed.SDKKey = envfactory.SDKKeyRep{Value: config.SDKKey("")} // undefined anchor + malformedRecoveryTest(t, makePatchEnvEvent(malformed), correctedEnv) + }) + + t.Run("anchor defined but absent from sdkKeys[] is malformed and recovers", func(t *testing.T) { + // A defined anchor (sdkKey.value) that does not appear in the authoritative sdkKeys[] array is + // treated as malformed by BuildAcceptedSet, just like an undefined anchor — exercise that variant + // through the stream manager. + malformed := testEnv1 + malformed.SDKKeys = []envfactory.ConcurrentKeyRep{{Key: "other", Value: "sdk-other-value"}} + malformedRecoveryTest(t, makeEnvPutEvent(malformed), correctedEnv) + }) +} + func TestMalformedJSONInEventCausesStreamRestart(t *testing.T) { t.Run("put", func(t *testing.T) { event := httphelpers.SSEEvent{Event: PutEvent, Data: malformedJSON} diff --git a/internal/envfactory/env_rep_test.go b/internal/envfactory/env_rep_test.go index 73c8e216..2499b045 100644 --- a/internal/envfactory/env_rep_test.go +++ b/internal/envfactory/env_rep_test.go @@ -188,3 +188,77 @@ func TestEnvironmentRepOldFormatNoArrays(t *testing.T) { require.Len(t, params.AcceptedMobileKeys, 1) assert.Equal(t, AcceptedMobileKey{Value: config.MobileKey("mob-default")}, params.AcceptedMobileKeys[0]) } + +// TestNewFormatPayloadDecodesIntoOldFormatStruct pins the wire-format additive guarantee directly. The +// local oldEnvironmentRep mirrors EnvironmentRep as it was before concurrent keys: only the singular +// sdkKey (with the legacy sdkKey.expiring rotation slot) and mobKey, and none of the new sdkKeys/ +// mobileKeys arrays or per-key expiry. Decoding a full new-format payload into it must succeed and yield +// exactly the singular values an old relay binary saw before the arrays existed — because the parse path +// uses the default JSON decoder (never DisallowUnknownFields), so the unknown new fields are ignored. +func TestNewFormatPayloadDecodesIntoOldFormatStruct(t *testing.T) { + // oldEnvironmentRep is the pre-concurrent-keys shape. Do not add sdkKeys/mobileKeys/expiry here — the + // whole point is that an old binary that never knew about them still decodes a new payload cleanly. + type oldEnvironmentRep struct { + EnvID config.EnvironmentID `json:"envID"` + EnvKey string `json:"envKey"` + EnvName string `json:"envName"` + MobKey config.MobileKey `json:"mobKey"` + ProjKey string `json:"projKey"` + ProjName string `json:"projName"` + SDKKey struct { + Value config.SDKKey `json:"value"` + Expiring struct { + Value config.SDKKey `json:"value"` + Timestamp ldtime.UnixMillisecondTime `json:"timestamp"` + } `json:"expiring"` + } `json:"sdkKey"` + DefaultTTL int `json:"defaultTtl"` + SecureMode bool `json:"secureMode"` + Version int `json:"version"` + } + + // A realistic full new-format payload: singular fields plus the legacy expiring slot, the new + // sdkKeys/mobileKeys arrays, and per-key expiry — everything a current backend can emit. + jsonStr := `{ + "envID": "68e5179e8307e4099c277e2a", + "envKey": "production", + "envName": "Production", + "mobKey": "mob-f41c", + "projKey": "my-project", + "projName": "My Project", + "sdkKey": { + "value": "sdk-anchor", + "expiring": { "value": "sdk-old-anchor", "timestamp": 1699000000000 } + }, + "sdkKeys": [ + { "key": "default-sdk", "value": "sdk-anchor" }, + { "key": "service-a", "value": "sdk-service-a", "expiry": 1700000000000 } + ], + "mobileKeys": [ + { "key": "mob-key-1", "value": "mob-f41c" }, + { "key": "mob-key-2", "value": "mob-second", "expiry": 1700000000000 } + ], + "defaultTtl": 5, + "secureMode": true, + "version": 26 + }` + + var rep oldEnvironmentRep + require.NoError(t, json.Unmarshal([]byte(jsonStr), &rep)) + + // The singular fields an old relay relies on populate exactly as before; the arrays are ignored. + assert.Equal(t, config.SDKKey("sdk-anchor"), rep.SDKKey.Value) + assert.Equal(t, config.SDKKey("sdk-old-anchor"), rep.SDKKey.Expiring.Value) + assert.Equal(t, ldtime.UnixMillisecondTime(1699000000000), rep.SDKKey.Expiring.Timestamp) + assert.Equal(t, config.MobileKey("mob-f41c"), rep.MobKey) + + // The remaining scalar fields also decode unchanged. + assert.Equal(t, config.EnvironmentID("68e5179e8307e4099c277e2a"), rep.EnvID) + assert.Equal(t, "production", rep.EnvKey) + assert.Equal(t, "Production", rep.EnvName) + assert.Equal(t, "my-project", rep.ProjKey) + assert.Equal(t, "My Project", rep.ProjName) + assert.Equal(t, 5, rep.DefaultTTL) + assert.True(t, rep.SecureMode) + assert.Equal(t, 26, rep.Version) +} diff --git a/internal/events/event_payload_regression_test.go b/internal/events/event_payload_regression_test.go index 2b2d0994..6d3e0a41 100644 --- a/internal/events/event_payload_regression_test.go +++ b/internal/events/event_payload_regression_test.go @@ -11,6 +11,7 @@ package events import ( + "net/http" "net/http/httptest" "testing" "time" @@ -18,6 +19,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/basictypes" st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/util" ldevents "github.com/launchdarkly/go-sdk-events/v3" helpers "github.com/launchdarkly/go-test-helpers/v3" @@ -32,53 +34,6 @@ func requireUpstreamRequest(t *testing.T, requestsCh <-chan httphelpers.HTTPRequ return helpers.RequireValue(t, requestsCh, time.Second) } -// TestAnalyticsUpstreamUsesAnchorCredential verifies that analytics events are forwarded -// upstream under the dispatcher's stored anchor credential, even when the incoming SDK -// request carries a different Authorization header. -func TestAnalyticsUpstreamUsesAnchorCredential(t *testing.T) { - eventRelayTest(t, st.EnvMain, config.EventsConfig{}, func(p eventRelayTestParams) { - headers := headersWithEventSchema(CurrentEventsSchemaVersion) - // Incoming request carries a non-anchor key — it must not reach the upstream. - headers.Set("Authorization", "sdk-non-anchor-key-must-not-reach-upstream") - - handler := p.dispatcher.GetHandler(basictypes.ServerSDK, ldevents.AnalyticsEventDataKind) - require.NotNil(t, handler) - w := httptest.NewRecorder() - handler(w, st.BuildRequest("POST", "/", []byte(eventPayloadForVerbatimOnly), headers)) - assert.Equal(t, 202, w.Result().StatusCode) - - p.dispatcher.flush() - r := requireUpstreamRequest(t, p.requestsCh) - - assert.Equal(t, string(st.EnvMain.Config.SDKKey), r.Request.Header.Get("Authorization"), - "analytics upstream must carry the anchor credential, not the incoming request credential") - }) -} - -// TestDiagnosticUpstreamProxiesIncomingCredential verifies that diagnostic events proxy -// the incoming request's Authorization header verbatim to the upstream, not the anchor. -func TestDiagnosticUpstreamProxiesIncomingCredential(t *testing.T) { - const sdkAuth = "sdk-original-diagnostic-client-auth" - - eventRelayTest(t, st.EnvMain, config.EventsConfig{}, func(p eventRelayTestParams) { - headers := headersWithEventSchema(0) - headers.Set("Authorization", sdkAuth) - - handler := p.dispatcher.GetHandler(basictypes.ServerSDK, ldevents.DiagnosticEventDataKind) - require.NotNil(t, handler) - w := httptest.NewRecorder() - handler(w, st.BuildRequest("POST", "/", []byte(eventPayloadForVerbatimOnly), headers)) - assert.Equal(t, 202, w.Result().StatusCode) - - r := requireUpstreamRequest(t, p.requestsCh) - - assert.Equal(t, sdkAuth, r.Request.Header.Get("Authorization"), - "diagnostic upstream must proxy the incoming Authorization header verbatim") - assert.NotEqual(t, string(st.EnvMain.Config.SDKKey), r.Request.Header.Get("Authorization"), - "diagnostic upstream must not use the anchor credential") - }) -} - // TestCredentialRoutingAfterReplaceCredential verifies that after ReplaceCredential is // called (anchor rotation), analytics events use the new anchor while diagnostic events // continue to proxy the original incoming authorization. @@ -135,3 +90,136 @@ func TestCredentialRoutingAfterReplaceCredential(t *testing.T) { "after rotation: diagnostic must not use the new anchor credential") }) } + +// headerSet builds a set of canonicalized header names for use as an allowed-delta list. +func headerSet(keys ...string) map[string]struct{} { + s := make(map[string]struct{}, len(keys)) + for _, k := range keys { + s[http.CanonicalHeaderKey(k)] = struct{}{} + } + return s +} + +// assertProxiedHeadersMatchExcept fails if any header present on either the incoming or the upstream +// request has a differing value, unless its (canonicalized) name is in allowedDelta. This pins that the +// forwarder passes headers through unchanged and diverges only on the enumerated transport/credential +// headers, catching any regression that silently drops, adds, or rewrites a header. +func assertProxiedHeadersMatchExcept(t *testing.T, incoming, upstream http.Header, allowedDelta map[string]struct{}) { + t.Helper() + checked := make(map[string]struct{}) + compare := func(h http.Header) { + for name := range h { + key := http.CanonicalHeaderKey(name) + if _, done := checked[key]; done { + continue + } + checked[key] = struct{}{} + if _, skip := allowedDelta[key]; skip { + continue + } + assert.Equalf(t, incoming.Get(key), upstream.Get(key), + "header %q must be forwarded unchanged", key) + } + } + compare(incoming) + compare(upstream) +} + +// TestAnalyticsUpstreamPayloadIsByteIdentical is the payload-regression form of the analytics routing +// test: beyond the credential decision, the forwarded body must be byte-for-byte identical to the +// incoming one, and every header must be forwarded unchanged except Authorization (swapped to the +// anchor) and the transport/compression headers the sender legitimately adds. +func TestAnalyticsUpstreamPayloadIsByteIdentical(t *testing.T) { + eventRelayTest(t, st.EnvMain, config.EventsConfig{}, func(p eventRelayTestParams) { + incomingBody := []byte(eventPayloadForVerbatimOnly) + incomingHeaders := headersWithEventSchema(CurrentEventsSchemaVersion) + incomingHeaders.Set(TagsHeader, "application-id/my-app application-version/1.2.3") + // The incoming request carries a non-anchor credential; it must not reach the upstream. + incomingHeaders.Set("Authorization", "sdk-non-anchor-key-must-not-reach-upstream") + + handler := p.dispatcher.GetHandler(basictypes.ServerSDK, ldevents.AnalyticsEventDataKind) + require.NotNil(t, handler) + w := httptest.NewRecorder() + handler(w, st.BuildRequest("POST", "/", incomingBody, incomingHeaders)) + assert.Equal(t, 202, w.Result().StatusCode) + + p.dispatcher.flush() + r := requireUpstreamRequest(t, p.requestsCh) + + // The upstream body is gzip-compressed; once decompressed it must equal the incoming bytes exactly. + uncompressed, err := util.DecompressGzipData(r.Body) + require.NoError(t, err) + assert.Equal(t, incomingBody, uncompressed, "the event payload body must be forwarded byte-for-byte") + + // Authorization is the one credential header the analytics path deliberately rewrites, swapping + // the incoming credential for the environment's anchor. + assert.Equal(t, string(st.EnvMain.Config.SDKKey), r.Request.Header.Get("Authorization"), + "analytics upstream must carry the anchor credential, not the incoming request credential") + + // Headers the analytics forwarder legitimately changes or adds, excluded from the + // "forwarded unchanged" comparison: + // Authorization - swapped to the environment anchor (asserted separately above) + // X-LaunchDarkly-Payload-ID - a fresh UUID the sender stamps on each analytics payload + // Content-Encoding - the sender gzip-compresses the payload + // Content-Length - reflects the compressed body, set by the transport + // Accept-Encoding - added by the Go HTTP transport + // User-Agent - Relay's own SDK-client User-Agent, from the base headers + allowedDelta := headerSet( + "Authorization", + "X-LaunchDarkly-Payload-ID", + "Content-Encoding", + "Content-Length", + "Accept-Encoding", + "User-Agent", + ) + assertProxiedHeadersMatchExcept(t, incomingHeaders, r.Request.Header, allowedDelta) + }) +} + +// TestDiagnosticUpstreamPayloadIsByteIdentical is the payload-regression form of the diagnostic routing +// test: the body must be byte-for-byte identical, and because the diagnostic path is a verbatim reverse +// proxy, every header including Authorization must be forwarded unchanged except the transport/compression +// headers the sender adds. +func TestDiagnosticUpstreamPayloadIsByteIdentical(t *testing.T) { + const sdkAuth = "sdk-original-diagnostic-client-auth" + + eventRelayTest(t, st.EnvMain, config.EventsConfig{}, func(p eventRelayTestParams) { + incomingBody := []byte(eventPayloadForVerbatimOnly) + incomingHeaders := headersWithEventSchema(0) + incomingHeaders.Set("Authorization", sdkAuth) + incomingHeaders.Set("User-Agent", "some-sdk/1.0.0") + // A custom header proves the diagnostic path forwards arbitrary request headers verbatim. + incomingHeaders.Set("X-Custom-Passthrough", "passthrough-value") + + handler := p.dispatcher.GetHandler(basictypes.ServerSDK, ldevents.DiagnosticEventDataKind) + require.NotNil(t, handler) + w := httptest.NewRecorder() + handler(w, st.BuildRequest("POST", "/", incomingBody, incomingHeaders)) + assert.Equal(t, 202, w.Result().StatusCode) + + r := requireUpstreamRequest(t, p.requestsCh) + + uncompressed, err := util.DecompressGzipData(r.Body) + require.NoError(t, err) + assert.Equal(t, incomingBody, uncompressed, "the diagnostic payload body must be forwarded byte-for-byte") + + // Unlike analytics, the diagnostic path proxies the incoming credential verbatim — the anchor is + // never substituted. + assert.Equal(t, sdkAuth, r.Request.Header.Get("Authorization"), + "diagnostic upstream must proxy the incoming Authorization header verbatim") + assert.NotEqual(t, string(st.EnvMain.Config.SDKKey), r.Request.Header.Get("Authorization"), + "diagnostic upstream must not use the anchor credential") + + // Only the transport/compression headers the sender adds may differ; everything else — including + // Authorization, User-Agent, and the custom header — is forwarded unchanged. + // Content-Encoding - the sender gzip-compresses the payload + // Content-Length - reflects the compressed body, set by the transport + // Accept-Encoding - added by the Go HTTP transport + allowedDelta := headerSet( + "Content-Encoding", + "Content-Length", + "Accept-Encoding", + ) + assertProxiedHeadersMatchExcept(t, incomingHeaders, r.Request.Header, allowedDelta) + }) +} diff --git a/internal/relayenv/env_context_reanchor_behavior_test.go b/internal/relayenv/env_context_reanchor_behavior_test.go new file mode 100644 index 00000000..a99a66ca --- /dev/null +++ b/internal/relayenv/env_context_reanchor_behavior_test.go @@ -0,0 +1,187 @@ +package relayenv + +// Permanent behavioral regression tests for re-anchoring. These pin three properties of the +// upstream-client swap: +// +// - an open downstream (client-side) connection survives a re-anchor and keeps receiving events, +// with the re-wired big-segment synchronizer driving its invalidations; +// - the new anchor's initial sync re-broadcasts a full "put" downstream, so a downstream SDK sees one +// duplicate put per re-anchor (tolerable — SDKs apply puts idempotently — but the swap must expect it); +// - httpconfig carries no baked-in SDK key other than the Authorization header the SDK sets per client, +// so it needs no re-wiring on a re-anchor. + +import ( + "net/http" + "sync" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/basictypes" + "github.com/launchdarkly/ld-relay/v8/internal/bigsegments" + "github.com/launchdarkly/ld-relay/v8/internal/httpconfig" + st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + "github.com/launchdarkly/ld-relay/v8/internal/store" + "github.com/launchdarkly/ld-relay/v8/internal/streams" + + "github.com/launchdarkly/eventsource" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + "github.com/launchdarkly/go-server-sdk/v7/ldcomponents" + "github.com/launchdarkly/go-server-sdk/v7/subsystems" + "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoretypes" + helpers "github.com/launchdarkly/go-test-helpers/v3" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// putCountingStreamUpdates is a streams.EnvStreamUpdates that counts the "all data" broadcasts it +// receives, so a test can observe how many full "put"s a sequence of store inits produces downstream. +type putCountingStreamUpdates struct { + mu sync.Mutex + allDataUpdates int +} + +func (r *putCountingStreamUpdates) SendAllDataUpdate(_ []ldstoretypes.Collection) { + r.mu.Lock() + r.allDataUpdates++ + r.mu.Unlock() +} + +func (r *putCountingStreamUpdates) SendSingleItemUpdate(_ ldstoretypes.DataKind, _ string, _ ldstoretypes.ItemDescriptor) { +} + +func (r *putCountingStreamUpdates) InvalidateClientSideState() {} + +func (r *putCountingStreamUpdates) allDataCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.allDataUpdates +} + +// TestReanchorDownstreamConnectionSurvives verifies that an open downstream client-side connection +// survives a re-anchor and keeps receiving events. The connection is keyed on the environment ID (a +// scoped credential) and is independent of the upstream SDK anchor key, so swapping the anchor must not +// disturb it. After the re-anchor the big-segment synchronizer has been rebuilt on the new anchor, so a +// big-segment update delivered on the current (rebuilt) synchronizer still pings the connected client. +func TestReanchorDownstreamConnectionSurvives(t *testing.T) { + envConfig := st.EnvClientSide.Config + + fakeBigSegmentStoreFactory := func(config.EnvConfig, config.Config, ldlog.Loggers) (bigsegments.BigSegmentStore, error) { + return bigsegments.NewNullBigSegmentStore(), nil + } + fakeSynchronizerFactory := &mockBigSegmentSynchronizerFactory{} + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + jsClientStreams := streams.NewStreamProvider(basictypes.JSClientPingStream, time.Hour, 0) + clientCh := make(chan *testclient.FakeLDClient, 10) + sdkStartedCh := make(chan EnvContext, 10) + env, err := NewEnvContext(EnvContextImplParams{ + Identifiers: EnvIdentifiers{ConfiguredName: st.EnvClientSide.Name}, + EnvConfig: envConfig, + AllConfig: config.Config{}, + BigSegmentStoreFactory: fakeBigSegmentStoreFactory, + BigSegmentSynchronizerFactory: fakeSynchronizerFactory.create, + ClientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientCh), + SDKBigSegmentsConfigFactory: ldcomponents.BigSegments( + st.ExistingInstance[subsystems.BigSegmentStore](&st.NoOpSDKBigSegmentStore{}), + ), + StreamProviders: []streams.StreamProvider{jsClientStreams}, + ConnectionMapper: mockConnectionMapper{}, + Loggers: mockLog.Loggers, + }, sdkStartedCh) + require.NoError(t, err) + defer env.Close() + + synchronizer := fakeSynchronizerFactory.synchronizer + require.NotNil(t, synchronizer) + + // Wait for the original anchor client and initialize the store so the client-side stream is ready. + <-sdkStartedCh + require.NoError(t, env.GetStore().Init(nil)) + + streamHandler := env.GetStreamHandler(jsClientStreams, envConfig.EnvID) + req, _ := http.NewRequest("GET", "", nil) + st.WithStreamRequest(t, req, streamHandler, func(eventCh <-chan eventsource.Event) { + initEvent := helpers.RequireValue(t, eventCh, time.Minute) + assert.Equal(t, "ping", initEvent.Event()) + if !helpers.AssertNoMoreValues(t, eventCh, 100*time.Millisecond) { + t.FailNow() + } + + // Re-anchor while the downstream connection is open. The re-anchor runs synchronously, so the + // new anchor's client is built and committed and the big-segment synchronizer is rebuilt by the + // time this returns. + now := time.Unix(1000, 0) + reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) + + // The synchronizer was rebuilt on the new anchor, so a post-re-anchor update arrives on the + // current (rebuilt) synchronizer, not the retired one (whose channel is now closed). + current := fakeSynchronizerFactory.synchronizer + require.NotSame(t, synchronizer, current, "the synchronizer was rebuilt on re-anchor") + current.updateCh <- bigsegments.UpdatesSummary{SegmentKeysUpdated: []string{"fake-segment-key"}} + pingEvent := helpers.RequireValue(t, eventCh, time.Second) + assert.Equal(t, "ping", pingEvent.Event(), "downstream connection should survive the re-anchor") + }) +} + +// TestReanchorInitialSyncRebroadcastsPut verifies that the new anchor's client performs its own initial +// sync when it comes up, re-broadcasting a full "put" to every connected downstream stream. From a +// downstream SDK's perspective this is a duplicate put on each re-anchor. It is tolerable — SDKs apply +// puts idempotently — but the re-anchor implementation must expect it; it is not a corruption. +func TestReanchorInitialSyncRebroadcastsPut(t *testing.T) { + rec := &putCountingStreamUpdates{} + adapter := store.NewSSERelayDataStoreAdapter(ldcomponents.InMemoryDataStore(), rec) + + // The original anchor client builds and performs its initial sync -> one downstream "put". + s1, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + require.NoError(t, s1.Init(st.AllData)) + require.Equal(t, 1, rec.allDataCount()) + + // Re-anchor: the new anchor's client performs its OWN initial sync (store handover hands it the same + // wrapper, but the new client still re-broadcasts a full put when it initializes). + s2, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + require.NoError(t, s2.Init(st.AllData)) + + assert.Equal(t, 2, rec.allDataCount(), "the new anchor's initial sync re-broadcasts a full put") +} + +// TestReanchorHTTPConfigIsKeyIndependent verifies that httpconfig carries no baked-in SDK key other than +// the Authorization default header, so a re-anchor needs no httpconfig re-wire. Relay injects the SDK +// HTTP config *builder* into the SDK config, and the SDK rebuilds the HTTP config with the new anchor key +// when it constructs the new client, so the Authorization header is set correctly for the new anchor +// automatically. The pre-built SDK HTTP config used for event and big-segment transport is +// key-independent except for that Authorization header, which those components set per request from +// their own credential rather than reading it from httpconfig. +func TestReanchorHTTPConfigIsKeyIndependent(t *testing.T) { + loggers := ldlog.NewDisabledLoggers() + key1 := config.SDKKey("sdk-key-one") + key2 := config.SDKKey("sdk-key-two") + + var proxy config.ProxyConfig + var httpC config.HTTPConfig + + c1, err := httpconfig.NewHTTPConfig(proxy, httpC, key1, "user-agent", loggers) + require.NoError(t, err) + c2, err := httpconfig.NewHTTPConfig(proxy, httpC, key2, "user-agent", loggers) + require.NoError(t, err) + + // The only key-dependent artifact is the Authorization default header on the pre-built SDK HTTP config. + assert.Equal(t, string(key1), c1.SDKHTTPConfig.DefaultHeaders.Get("Authorization")) + assert.Equal(t, string(key2), c2.SDKHTTPConfig.DefaultHeaders.Get("Authorization")) + + // Everything else (proxy settings, user agent, and the rest of the default headers) is identical and + // key-independent. + h1 := c1.SDKHTTPConfig.DefaultHeaders.Clone() + h2 := c2.SDKHTTPConfig.DefaultHeaders.Clone() + h1.Del("Authorization") + h2.Del("Authorization") + assert.Equal(t, h1, h2, "non-auth default headers are key-independent") + assert.Equal(t, c1.ProxyConfig, c2.ProxyConfig, "proxy config is key-independent") +} diff --git a/internal/relayenv/env_context_reanchor_bigsegment_test.go b/internal/relayenv/env_context_reanchor_bigsegment_test.go index 5f06669a..4fa18f20 100644 --- a/internal/relayenv/env_context_reanchor_bigsegment_test.go +++ b/internal/relayenv/env_context_reanchor_bigsegment_test.go @@ -5,25 +5,20 @@ package relayenv // not yet started" case; these cover the started-continues, rollback, and not-configured cases. import ( - "net/http" "testing" "time" "github.com/launchdarkly/ld-relay/v8/config" - "github.com/launchdarkly/ld-relay/v8/internal/basictypes" "github.com/launchdarkly/ld-relay/v8/internal/bigsegments" "github.com/launchdarkly/ld-relay/v8/internal/sdks" st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" - "github.com/launchdarkly/ld-relay/v8/internal/streams" - "github.com/launchdarkly/eventsource" "github.com/launchdarkly/go-sdk-common/v3/ldlog" "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" ld "github.com/launchdarkly/go-server-sdk/v7" "github.com/launchdarkly/go-server-sdk/v7/ldcomponents" "github.com/launchdarkly/go-server-sdk/v7/subsystems" - helpers "github.com/launchdarkly/go-test-helpers/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -154,59 +149,6 @@ func TestReanchorBigSegmentSync_NotConfiguredIsNoOp(t *testing.T) { assert.Equal(t, reanchorTestKey2, env.(*envContextImpl).keyRotator.AnchorKey(), "the SDK re-anchor still committed") } -// TestReanchorBigSegmentSync_NewSyncDrivesClientSideInvalidation is the end-to-end integration case -// (SDK-2543 AC): with a client-side stream connected across a re-anchor, a big-segment update delivered -// on the NEW synchronizer must still ping the connected client -- proving the re-wired synchronizer's -// update consumer is active and drives client-side invalidation. -func TestReanchorBigSegmentSync_NewSyncDrivesClientSideInvalidation(t *testing.T) { - envConfig := st.EnvClientSide.Config - capturing := &capturingBigSegmentSynchronizerFactory{} - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - jsClientStreams := streams.NewStreamProvider(basictypes.JSClientPingStream, time.Hour, 0) - sdkStartedCh := make(chan EnvContext, 1) - clientCh := make(chan *testclient.FakeLDClient, 10) - env, err := NewEnvContext(EnvContextImplParams{ - Identifiers: EnvIdentifiers{ConfiguredName: st.EnvMain.Name}, - EnvConfig: envConfig, - AllConfig: config.Config{}, - BigSegmentStoreFactory: nullBigSegmentStoreFactory, - BigSegmentSynchronizerFactory: capturing.create, - ClientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientCh), - SDKBigSegmentsConfigFactory: ldcomponents.BigSegments( - st.ExistingInstance[subsystems.BigSegmentStore](&st.NoOpSDKBigSegmentStore{}), - ), - StreamProviders: []streams.StreamProvider{jsClientStreams}, - ConnectionMapper: mockConnectionMapper{}, - Loggers: mockLog.Loggers, - }, sdkStartedCh) - require.NoError(t, err) - defer env.Close() - - <-sdkStartedCh - _ = env.GetStore().Init(nil) // client-side endpoint only pings once the store is initialized - oldSync := capturing.latest() - - streamHandler := env.GetStreamHandler(jsClientStreams, envConfig.EnvID) - req, _ := http.NewRequest("GET", "", nil) - st.WithStreamRequest(t, req, streamHandler, func(eventCh <-chan eventsource.Event) { - initEvent := helpers.RequireValue(t, eventCh, time.Minute) - require.Equal(t, "ping", initEvent.Event()) - helpers.AssertNoMoreValues(t, eventCh, 100*time.Millisecond) - - // Re-anchor mid-subscription; the synchronizer is rebuilt on the new anchor. - reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) - newSync := capturing.latest() - require.NotSame(t, oldSync, newSync, "the synchronizer was rebuilt on re-anchor") - - // A big-segment update on the NEW synchronizer pings the still-connected client-side stream. - newSync.updateCh <- bigsegments.UpdatesSummary{SegmentKeysUpdated: []string{"seg"}} - pingEvent := helpers.RequireValue(t, eventCh, time.Second) - assert.Equal(t, "ping", pingEvent.Event()) - }) -} - // reanchorTestKey3 is a third anchor SDK key, used to drive A->B->C sequential re-anchors. const reanchorTestKey3 = config.SDKKey("reanchor-poc-new-anchor-3") @@ -326,3 +268,47 @@ func TestReanchorBigSegmentSync_ConcurrentStoreUpdateDuringReanchorIsRaceFree(t assert.Equal(t, 2, count, "the synchronizer was recreated on re-anchor") assert.Equal(t, reanchorTestKey2, sdkKey, "on the new anchor key") } + +// TestReanchorBigSegmentSync_RepromoteInGraceFormerAnchorRewires re-anchors A->B, then B->A while A is +// still accepted (an in-grace former anchor whose client was closed when its demotion committed). +// Promoting a previously-accepted key must re-wire big segments identically to a brand-new key: a fresh +// synchronizer bound to A is created and Started (a segment already exists), and B's synchronizer is +// Closed. This pins that the "previously-accepted key" promotion path does not shortcut the big-segment +// re-wire. +func TestReanchorBigSegmentSync_RepromoteInGraceFormerAnchorRewires(t *testing.T) { + envConfig := st.EnvMain.Config + capturing := &capturingBigSegmentSynchronizerFactory{} + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + env := newBigSegmentTestEnv(t, nullBigSegmentStoreFactory, + testclient.FakeLDClientFactoryWithChannel(true, clientCh), capturing, mockLog.Loggers) + defer env.Close() + envImpl := env.(*envContextImpl) + + envImpl.setBigSegmentsExist() + syncA := capturing.latest() + require.True(t, syncA.isStarted(), "the synchronizer is started once a big segment exists") + + // A -> B. A stays accepted in its grace period; its client is closed at commit. + reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) + syncB := capturing.latest() + require.NotSame(t, syncA, syncB) + require.True(t, syncA.isClosed(), "A's synchronizer is closed after A->B") + require.True(t, syncB.isStarted(), "B's synchronizer is started (a segment already existed)") + + // B -> A. A is still in the accepted set (its mappings survived the demotion) but has no client, so a + // fresh client is built and the big-segment sync must be re-wired onto A just like a brand-new key. + reanchor(t, env, envConfig.SDKKey, reanchorTestKey2, time.Unix(1000, 0)) + syncARepromoted := capturing.latest() + assert.NotSame(t, syncB, syncARepromoted, "re-promoting A builds a THIRD synchronizer instance") + assert.NotSame(t, syncA, syncARepromoted, "and a fresh instance, not the retired original A synchronizer") + + count, sdkKey := capturing.snapshot() + assert.Equal(t, 3, count, "one synchronizer per anchor commit: A, B, A-again") + assert.Equal(t, envConfig.SDKKey, sdkKey, "the third synchronizer is bound to the re-promoted key A") + assert.True(t, syncARepromoted.isStarted(), "the re-promoted anchor's synchronizer is Started (a segment exists)") + assert.True(t, syncB.isClosed(), "B's synchronizer is Closed on the re-promotion") + assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "the SDK anchor is back on A") +} diff --git a/internal/relayenv/env_context_reanchor_close_test.go b/internal/relayenv/env_context_reanchor_close_test.go new file mode 100644 index 00000000..7ef5a1a6 --- /dev/null +++ b/internal/relayenv/env_context_reanchor_close_test.go @@ -0,0 +1,109 @@ +package relayenv + +// Regression test for tearing down an environment while a re-anchor's client build is in flight. The +// re-anchor releases c.mu around the (potentially slow, SDK-init-timeout-bounded) client build, so a +// concurrent Close() can run its teardown during that window. When the build then completes, the +// re-anchor must observe that the env is closed, discard and close the freshly-built client rather than +// installing it, and leave no client behind. + +import ( + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" + st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + "github.com/launchdarkly/ld-relay/v8/internal/util" + + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + ld "github.com/launchdarkly/go-server-sdk/v7" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const reanchorCloseNewAnchor = config.SDKKey("reanchor-close-new-anchor") + +// TestReanchorClosedDuringInFlightBuild wedges the new anchor's synchronous client build, calls Close() +// from another goroutine, and then releases the build. Close must return within a generous deadline +// (it does not block on the wedged build — the build runs without c.mu held). The late-built client is +// discarded and closed rather than installed, no client remains, and a subsequent GetClient() returns +// nil without panicking. +// +// The build is bounded by the SDK init timeout in production; here the gate release unblocks it +// deterministically. This test is run under -race to catch any unsynchronized access between the +// Close teardown and the re-anchor's post-build lock re-acquisition. +func TestReanchorClosedDuringInFlightBuild(t *testing.T) { + envConfig := st.EnvMain.Config + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + clientCh := make(chan *testclient.FakeLDClient, 10) + inner := testclient.FakeLDClientFactoryWithChannel(true, clientCh) + + // The new anchor's build blocks on gate; entered signals the build has reached the factory (the + // re-anchor is mid-flight, pre-commit, and holds no c.mu while it waits here). + gate := make(chan struct{}) + entered := make(chan struct{}, 1) + gatedFactory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == reanchorCloseNewAnchor { + entered <- struct{}{} + <-gate + } + return inner(sdkKey, cfg, timeout) + } + + readyCh := make(chan EnvContext, 1) + env := makeBasicEnv(t, envConfig, gatedFactory, mockLog.Loggers, readyCh) + defer env.Close() // idempotent; the test closes explicitly below, this guards early failures + + require.Equal(t, env, requireEnvReady(t, readyCh)) + originalClient := requireClientReady(t, clientCh) + require.Eventually(t, func() bool { return env.GetClient() == originalClient }, time.Second, 10*time.Millisecond) + + // Drive the re-anchor on a background goroutine; it blocks in the gated build. + start := time.Unix(2000, 0) + set, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: reanchorCloseNewAnchor}). + WithSDKKey(credential.SDKKeyParams{Value: envConfig.SDKKey, Expiry: util.PtrOrNil(start.Add(time.Hour))}). + Build() + require.NoError(t, err) + reconcileDone := make(chan struct{}) + go func() { + defer close(reconcileDone) + env.(*envContextImpl).reconcileCredentials(set, start) + }() + + <-entered // the build is wedged: the re-anchor is mid-flight, pre-commit. + + // Close from another goroutine. It does not hold reconcileMu, and the re-anchor released c.mu around + // the build, so Close can proceed to tear down the env while the build is still blocked. + closeDone := make(chan error, 1) + go func() { closeDone <- env.Close() }() + select { + case err := <-closeDone: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("Close did not return while the re-anchor build was wedged") + } + + // Release the wedged build. The late client is built and returned to the re-anchor, which finds the + // env closed and discards + closes it rather than installing it. + close(gate) + <-reconcileDone + + // The late-built client was closed (its close channel fires) and never installed. + lateClient := requireClientReady(t, clientCh) + lateClient.AwaitClose(t, time.Second) + + envImpl := env.(*envContextImpl) + envImpl.mu.RLock() + remaining := len(envImpl.clients) + envImpl.mu.RUnlock() + assert.Equal(t, 0, remaining, "no client remains installed after Close during an in-flight re-anchor") + + // GetClient after Close returns nil without panicking. + assert.Nil(t, env.GetClient(), "GetClient returns nil after Close, with no panic") +} diff --git a/internal/sharedtest/configsource/rac_mock.go b/internal/sharedtest/configsource/rac_mock.go index 976baf74..3a394f35 100644 --- a/internal/sharedtest/configsource/rac_mock.go +++ b/internal/sharedtest/configsource/rac_mock.go @@ -40,6 +40,28 @@ func NewRACMock(t testing.TB, initialEvent *httphelpers.SSEEvent) *RACMock { return m } +// NewRACMockWithReconnect creates a RACMock that serves firstEvent to the first client that connects +// and reconnectEvent to the next client — modeling a stream that a client restarts and reconnects to. +// This supports the design's malformed-payload recovery (§9): a rejected patch forces Relay to restart +// its config stream, and the backend serves a fresh, corrected put on the reconnection. +// +// Send delivers to the first connection; use it to push the event that forces the restart (e.g. the +// malformed patch). initialEvent replay applies per handler, so the first connection sees firstEvent +// and the reconnection sees reconnectEvent. Cleanup is registered with t.Cleanup. +func NewRACMockWithReconnect(t testing.TB, firstEvent, reconnectEvent *httphelpers.SSEEvent) *RACMock { + firstHandler, firstStream := httphelpers.SSEHandler(firstEvent) + reconnectHandler, _ := httphelpers.SSEHandler(reconnectEvent) + handler := httphelpers.SequentialHandler(firstHandler, reconnectHandler) + server := httptest.NewServer(handler) + m := &RACMock{ + URL: server.URL, + server: server, + stream: firstStream, + } + t.Cleanup(m.Close) + return m +} + // Enqueue queues an event to be delivered to the next client that connects. Use this before Relay // has connected to ensure the event is not dropped. func (m *RACMock) Enqueue(event httphelpers.SSEEvent) { diff --git a/relay/autoconfig_actions_test.go b/relay/autoconfig_actions_test.go index 8ca7301f..be7fe78f 100644 --- a/relay/autoconfig_actions_test.go +++ b/relay/autoconfig_actions_test.go @@ -8,6 +8,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/internal/envfactory" c "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" "github.com/launchdarkly/go-configtypes" @@ -43,6 +44,24 @@ func autoConfTest( config c.Config, initialEvent *httphelpers.SSEEvent, action func(p autoConfTestParams), +) { + autoConfTestWithClientFactory(t, config, initialEvent, + func(createdCh chan<- *testclient.FakeLDClient) sdks.ClientFactoryFunc { + return testclient.FakeLDClientFactoryWithChannel(true, createdCh) + }, action) +} + +// autoConfTestWithClientFactory is autoConfTest with a caller-supplied SDK client factory, so a test +// can inject a factory that fails or hangs for specific keys (e.g. to exercise the re-anchor +// init-failure rollback through the real RAC handler). makeClientFactory receives the channel that +// created clients are reported on; the usual body wraps testclient.FakeLDClientFactoryWithChannel and +// special-cases only the keys it wants to treat differently, forwarding the rest to the healthy factory. +func autoConfTestWithClientFactory( + t *testing.T, + config c.Config, + initialEvent *httphelpers.SSEEvent, + makeClientFactory func(createdCh chan<- *testclient.FakeLDClient) sdks.ClientFactoryFunc, + action func(p autoConfTestParams), ) { mockLog := ldlogtest.NewMockLog() defer mockLog.DumpIfTestFailed(t) @@ -78,7 +97,7 @@ func autoConfTest( relay, err := newRelayInternal(config, relayInternalOptions{ loggers: mockLog.Loggers, - clientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientsCreatedCh), + clientFactory: makeClientFactory(clientsCreatedCh), }) if err != nil { panic(err) diff --git a/relay/concurrent_keys_defensive_test.go b/relay/concurrent_keys_defensive_test.go new file mode 100644 index 00000000..99d83695 --- /dev/null +++ b/relay/concurrent_keys_defensive_test.go @@ -0,0 +1,94 @@ +package relay + +// Defensive-behavior integration tests for the RAC config stream: a malformed credential payload must +// preserve the previous accepted set and force a stream restart, and the fresh state the backend +// serves on the reconnection must then be applied. This is the relay-level twin of the unit-level +// reconnect coverage in internal/autoconfig; it verifies the whole loop end-to-end through Relay's +// downstream auth surface. + +import ( + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/configsource" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + "github.com/launchdarkly/go-configtypes" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + + "github.com/stretchr/testify/require" +) + +// A malformed patch (anchor absent from sdkKeys[]) is rejected without being applied: the previously +// accepted credentials keep authenticating and no new key leaks in. The rejection forces the config +// stream to restart, and on the reconnection the backend serves a corrected put whose new key then +// authenticates — completing the preserve-then-recover loop from the design's malformed-payload policy. +func TestConcurrentKeysRAC_MalformedPayloadRecoversAfterReconnect(t *testing.T) { + firstPut := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + + // The corrected state the backend serves on the reconnection: the original two keys plus a new one. + correctedPut := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-sdk", Value: string(anchorSDKKey)}, + {Key: "extra-sdk", Value: string(extraSDKKey)}, + {Key: "added-sdk", Value: string(addedSDKKey)}, + }, + defaultMobileKeyReps(), + 3, + )) + + racMock := configsource.NewRACMockWithReconnect(t, &firstPut, &correctedPut) + + cfg := config.Config{AutoConfig: config.AutoConfigConfig{Key: testAutoConfKey}} + cfg.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(racMock.URL) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + relay, err := newRelayInternal(cfg, relayInternalOptions{ + loggers: mockLog.Loggers, + clientFactory: testclient.FakeLDClientFactory(true), + }) + require.NoError(t, err) + defer relay.Close() + + h := relayTestHelper{t: t, relay: relay} + env := h.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + // Baseline: the two original keys authenticate; the corrected-put key is not accepted yet. + h.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + h.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + h.assertSDKEndpointsAvailability(false, addedSDKKey, "", "") + + // Send a structurally malformed patch: the anchor (anchorSDKKey) is absent from sdkKeys[]. Building + // the raw rep this way makes credential-set validation fail at the stream parse boundary. + malformed := multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{{Key: "extra-sdk", Value: string(extraSDKKey)}}, + defaultMobileKeyReps(), + 2, + ) + racMock.Send(configsource.MakeAutoConfigPatchEvent(malformed)) + + // The malformed payload is rejected and logged; the previous accepted set is preserved (unchanged). + require.Eventually(t, func() bool { + return mockLog.HasMessageMatch(ldlog.Error, "[Mm]alformed credential payload") + }, 5*time.Second, 10*time.Millisecond, "malformed payload was not rejected with a structured error") + h.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + h.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + + // Recovery: the rejection restarted the stream, and on the reconnection the backend's corrected put + // applies — the new key now authenticates. + require.Eventually(t, func() bool { + _, err := relay.getEnvironment(sdkauth.New(addedSDKKey)) + return err == nil + }, 5*time.Second, 10*time.Millisecond, "corrected put was not applied after the reconnect") + + h.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + h.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + h.assertSDKEndpointsAvailability(true, addedSDKKey, "", "") +} diff --git a/relay/concurrent_keys_lifecycle_test.go b/relay/concurrent_keys_lifecycle_test.go new file mode 100644 index 00000000..e2783063 --- /dev/null +++ b/relay/concurrent_keys_lifecycle_test.go @@ -0,0 +1,212 @@ +package relay + +// Key-lifecycle integration tests that observe live downstream SSE streams (real SDK clients via the +// offline harness, or a dummy client + RAC mock) rather than just the auth layer: mixed reconcile +// updates, revocation by omission, and sibling-stream continuity during a targeted disconnect. + +import ( + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/filedata" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/configsource" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + "github.com/launchdarkly/eventsource" + "github.com/launchdarkly/go-configtypes" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// base64 of {"key":"userkey","kind":"user"} — a valid mobile-eval context path. +const mobileEvalContextPath = "/meval/eyJrZXkiOiJ1c2Vya2V5Iiwia2luZCI6InVzZXIifQ==" + +// reanchoredArchiveEnv builds an offline ArchiveEnvironment whose anchor is newAnchor and whose +// accepted SDK key set is exactly sdkKeys, keeping the standard mobile keys. Used to model an offline +// reload that re-anchors and rewrites the server-key set in one step. +func reanchoredArchiveEnv(newAnchor config.SDKKey, sdkKeys []envfactory.AcceptedSDKKey) filedata.ArchiveEnvironment { + return filedata.ArchiveEnvironment{ + Params: envfactory.EnvironmentParams{ + EnvID: multiKeyEnvID, + SDKKey: newAnchor, + MobileKey: anchorMobileKey, + AcceptedSDKKeys: sdkKeys, + AcceptedMobileKeys: defaultAcceptedMobileKeys(), + Identifiers: multiKeyIdentifiers, + }, + SDKData: multiKeySDKData(), + } +} + +// A single offline reload that adds a key, re-anchors to a brand-new key, and removes the old extra +// key all at once. The end state is deterministic (add -> re-anchor -> remove): the added key, the new +// anchor, and the retained mobile keys authenticate; the old anchor and the removed key do not. In +// offline mode the re-anchor builds no new upstream client (the single file-data client keeps serving), +// and a downstream stream open on a retained credential survives the reload undisturbed. +func TestConcurrentKeysOffline_MixedUpdateAddsReanchorsAndRemovesInOneReload(t *testing.T) { + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) + + initialClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, initialClient.Key) + env := p.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + // Baseline: anchor, extra SDK key, and both mobile keys authenticate. + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + + // Hold a downstream stream on a retained credential (a mobile key, unaffected by the SDK + // re-anchor) across the reload. + req := sharedtest.BuildRequestWithAuth("GET", mobileEvalContextPath, anchorMobileKey, nil) + sharedtest.WithStreamRequest(t, req, p.relay, func(eventCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, eventCh, "ping", 5*time.Second) + + // One reload doing three things at once: add addedSDKKey, re-anchor to the brand-new + // rotatedAnchorSDKKey, and drop the old anchor and extraSDKKey (omitted from the new set). + p.updateHandler.UpdateEnvironment(reanchoredArchiveEnv(rotatedAnchorSDKKey, + []envfactory.AcceptedSDKKey{{Value: rotatedAnchorSDKKey}, {Value: addedSDKKey}})) + + // The retained mobile-key stream is not disconnected by the reload. + assertStreamStaysOpen(t, eventCh, 300*time.Millisecond) + }) + + // Offline re-anchor stands up no new upstream client. + p.shouldNotCreateClient(200 * time.Millisecond) + + // End state: old anchor and removed extra key are gone; the new anchor, the added key, and the + // retained mobile keys authenticate. + awaitCredentialRemoved(t, p.relay, anchorSDKKey) + awaitCredentialRemoved(t, p.relay, extraSDKKey) + p.assertSDKEndpointsAvailability(true, rotatedAnchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, addedSDKKey, "", "") + p.assertSDKEndpointsAvailability(true, "", extraMobileKey, "") + p.assertSDKEndpointsAvailability(false, anchorSDKKey, "", "") + p.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") + }) +} + +// A non-anchor key omitted from the next RAC patch is revoked immediately (not on a grace timer), and +// a downstream SDK connected on that key is disconnected as part of the revocation. The anchor, which +// the patch retains, keeps authenticating. Uses a real (dummy) client + RAC mock so there is a live +// stream to observe being torn down. +func TestConcurrentKeysRAC_ConnectedStreamClosedWhenKeyRevokedByOmission(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + racMock := configsource.NewRACMock(t, &putEvent) + + cfg := config.Config{AutoConfig: config.AutoConfigConfig{Key: testAutoConfKey}} + cfg.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(racMock.URL) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + relay, err := newRelayInternal(cfg, relayInternalOptions{ + loggers: mockLog.Loggers, + clientFactory: testclient.CreateDummyClient, + }) + require.NoError(t, err) + defer relay.Close() + + h := relayTestHelper{t: t, relay: relay} + env := h.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + // Connect a downstream SDK on the non-anchor key that the next patch will omit. + req := sharedtest.BuildRequestWithAuth("GET", "/all", extraSDKKey, nil) + sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) + + // Revoke by omission: a patch that carries only the anchor SDK key (extraSDKKey dropped), keeping + // the mobile keys. The reconcile revokes the omitted key now rather than on a grace timer. + racMock.Send(configsource.MakeAutoConfigPatchEvent(multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{{Key: "anchor-sdk", Value: string(anchorSDKKey)}}, + defaultMobileKeyReps(), + 2, + ))) + + // The revoked key's open stream is disconnected. + awaitStreamClosed(t, eventCh, 5*time.Second) + }) + + awaitCredentialRemoved(t, relay, extraSDKKey) + h.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") + h.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) +} + +// The offline-reload twin of the RAC revocation-by-omission case: a key dropped from the reloaded +// archive is revoked immediately and its connected downstream SDK is disconnected, while the retained +// anchor keeps authenticating. +func TestConcurrentKeysOffline_ConnectedStreamClosedWhenKeyRevokedByOmission(t *testing.T) { + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) + _ = p.awaitClient() + env := p.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + req := sharedtest.BuildRequestWithAuth("GET", "/all", extraSDKKey, nil) + sharedtest.WithStreamRequest(t, req, p.relay, func(eventCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) + + // Reload with the non-anchor key omitted: it is revoked immediately. + p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv( + []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}}, + defaultAcceptedMobileKeys(), + )) + + awaitStreamClosed(t, eventCh, 5*time.Second) + }) + + awaitCredentialRemoved(t, p.relay, extraSDKKey) + p.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + }) +} + +// When one key expires, the disconnect must be targeted: only that key's downstream SDKs drop. A stream +// held on the anchor stays connected throughout the expiry window while a concurrently-open stream on +// the expiring non-anchor key is torn down. Uses the offline harness (real client that serves stream +// data) with two simultaneous downstream connections on the same environment. +func TestConcurrentKeysOffline_SiblingStreamSurvivesWhileExpiringKeyDisconnects(t *testing.T) { + cfg := config.Config{} + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) + offlineModeTest(t, cfg, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) + _ = p.awaitClient() + env := p.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + // Open a stream on the anchor — the sibling that must stay connected. + anchorReq := sharedtest.BuildRequestWithAuth("GET", "/all", anchorSDKKey, nil) + sharedtest.WithStreamRequest(t, anchorReq, p.relay, func(anchorCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, anchorCh, "put", 5*time.Second) + + // Concurrently open a second stream on the non-anchor key that we will expire. + expiringReq := sharedtest.BuildRequestWithAuth("GET", "/all", extraSDKKey, nil) + sharedtest.WithStreamRequest(t, expiringReq, p.relay, func(expiringCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, expiringCh, "put", 5*time.Second) + + // Give the non-anchor key a near-future expiry; the anchor stays permanent. + expiry := time.Now().Add(100 * time.Millisecond) + p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv( + []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}, {Value: extraSDKKey, Expiry: expiry}}, + defaultAcceptedMobileKeys(), + )) + + // Across the expiry window the anchor sibling's stream stays open (the expiring key's + // stream is being torn down on its own channel during this same window)... + assertStreamStaysOpen(t, anchorCh, 300*time.Millisecond) + // ...and the expiring key's stream is confirmed disconnected. + awaitStreamClosed(t, expiringCh, 5*time.Second) + }) + }) + + // After the expiry: the dropped key no longer authenticates; the anchor sibling still does. + p.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + }) +} diff --git a/relay/concurrent_keys_reanchor_test.go b/relay/concurrent_keys_reanchor_test.go new file mode 100644 index 00000000..11460c6e --- /dev/null +++ b/relay/concurrent_keys_reanchor_test.go @@ -0,0 +1,210 @@ +package relay + +// Re-anchor integration tests driven through the real RAC handler (autoConfTest), covering the +// failure and default-rotation paths that the auth-focused tests in concurrent_keys_auth_test.go do +// not: init-failure rollback (and subsequent recovery), and the backend's default-rotation array +// shape that grace-demotes the old anchor. + +import ( + "encoding/json" + "errors" + "net/http" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/api" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/configsource" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + "github.com/launchdarkly/eventsource" + "github.com/launchdarkly/go-configtypes" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + ld "github.com/launchdarkly/go-server-sdk/v7" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// When the newly-designated anchor's SDK client fails to initialize, the re-anchor rolls back: the +// previous anchor and its siblings keep authenticating, the failed key is rejected, the environment +// stays connected, and a structured error is logged. The rollback must also leave the environment +// recoverable — a later payload rotating to a healthy key re-anchors cleanly. This drives the whole +// sequence through the RAC handler with a client factory that refuses to initialize one specific key. +func TestConcurrentKeysRAC_ReanchorInitFailureRollsBackAndRecovers(t *testing.T) { + // The healthy recovery anchor rotated to after the failed rotation is rolled back. + const recoveryAnchorSDKKey = config.SDKKey("sdk-recovery-anchor") + + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + + // Fail to initialize only the first rotated anchor; every other key (the original anchor, the + // healthy recovery anchor) builds normally and is reported on the created-clients channel. + makeFactory := func(createdCh chan<- *testclient.FakeLDClient) sdks.ClientFactoryFunc { + healthy := testclient.FakeLDClientFactoryWithChannel(true, createdCh) + return func(key config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if key == rotatedAnchorSDKKey { + return nil, errors.New("re-anchor: new client init refused") + } + return healthy(key, cfg, timeout) + } + } + + autoConfTestWithClientFactory(t, testAutoConfDefaultConfig, &putEvent, makeFactory, func(p autoConfTestParams) { + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) + _ = p.awaitEnvironment(multiKeyEnvID) + + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + + // Rotate the anchor to the key whose client refuses to initialize. The synchronous re-anchor + // build fails, so the reconcile rolls back the anchor change. + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(rotatedAnchorRep(rotatedAnchorSDKKey, 2))) + + // Wait for the rollback to settle: the structured error is logged and the failed new anchor's + // briefly-registered mappings are torn down. The error is logged before the mapping teardown, so + // requiring both proves we observe the fully rolled-back state (not the mid-swap registration window). + require.Eventually(t, func() bool { + _, err := p.relay.getEnvironment(sdkauth.New(rotatedAnchorSDKKey)) + return err != nil && p.mockLog.HasMessageMatch(ldlog.Error, "Re-anchor to SDK key .* failed") + }, 5*time.Second, 10*time.Millisecond, "re-anchor init failure did not roll back with a structured error") + + // Previous anchor and its sibling still authenticate; the failed new anchor is rejected. + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(false, rotatedAnchorSDKKey, "", "") + + // The environment is still connected — the preserved anchor's client keeps serving. + assert.Equal(t, "connected", requireEnvStatus(t, p.relay).Status) + + // The failed rotation built nothing that stuck around. + p.shouldNotCreateClient(200 * time.Millisecond) + + // Recovery: rotate to a healthy key. The re-anchor now commits — a new client comes up, the new + // anchor authenticates, and the previous anchor's client is torn down. + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(rotatedAnchorRep(recoveryAnchorSDKKey, 3))) + + recoveryClient := p.awaitClient() + assert.Equal(t, recoveryAnchorSDKKey, recoveryClient.Key) + anchorClient.AwaitClose(t, 5*time.Second) + awaitCredentialRemoved(t, p.relay, anchorSDKKey) + + p.assertSDKEndpointsAvailability(true, recoveryAnchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(false, anchorSDKKey, "", "") + p.assertSDKEndpointsAvailability(false, rotatedAnchorSDKKey, "", "") + }) +} + +// requireEnvStatus fetches the /status endpoint and returns the single environment's status rep, +// failing the test if the response does not describe exactly one environment. +func requireEnvStatus(t *testing.T, relay *Relay) api.EnvironmentStatusRep { + t.Helper() + req, _ := http.NewRequest("GET", "/status", nil) + result, body := sharedtest.DoRequest(req, relay) + require.Equal(t, http.StatusOK, result.StatusCode) + var status api.StatusRep + require.NoError(t, json.Unmarshal(body, &status)) + require.Len(t, status.Environments, 1) + for _, envStatus := range status.Environments { + return envStatus + } + return api.EnvironmentStatusRep{} +} + +// defaultRotationRep builds the RAC EnvironmentRep the backend emits on a default rotation: sdkKey.value +// flips to newAnchor (permanent, no expiry) and the demoted old anchor stays in sdkKeys[] carrying a +// grace expiry. Mobile keys are unchanged. +func defaultRotationRep(newAnchor, demotedAnchor config.SDKKey, demotedExpiry int64, version int) envfactory.EnvironmentRep { + return envfactory.EnvironmentRep{ + EnvID: multiKeyEnvID, + EnvKey: multiKeyIdentifiers.EnvKey, + EnvName: multiKeyIdentifiers.EnvName, + ProjKey: multiKeyIdentifiers.ProjKey, + ProjName: multiKeyIdentifiers.ProjName, + SDKKey: envfactory.SDKKeyRep{Value: newAnchor}, + MobKey: anchorMobileKey, + SDKKeys: []envfactory.ConcurrentKeyRep{ + {Key: "new-anchor-sdk", Value: string(newAnchor)}, + {Key: "old-anchor-sdk", Value: string(demotedAnchor), Expiry: msPtr(demotedExpiry)}, + }, + MobileKeys: defaultMobileKeyReps(), + Version: version, + } +} + +// On a default rotation the backend flips sdkKey.value to a promoted key and demotes the old anchor with +// a grace expiry, all in one patch (the array carries [new (no expiry), old (expiry)]). Relay must +// re-anchor to the new key while keeping the demoted old anchor serving through its grace window — a +// downstream stream opened on the old anchor before the rotation survives the swap — and then, once the +// grace expiry passes, drop the old anchor and disconnect that stream. The new anchor authenticates +// throughout. Uses a real (dummy) client + RAC mock so there is a live stream to observe (FakeLDClient +// never serves a stream body). +func TestConcurrentKeysRAC_DefaultRotationGraceDemotesOldAnchor(t *testing.T) { + // The initial env has a single (anchor) SDK key, so the default-rotation array is exactly + // [new (no expiry), old (expiry)] — the minimal shape the backend emits on a default rotation. + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{{Key: "anchor-sdk", Value: string(anchorSDKKey)}}, + defaultMobileKeyReps(), 1)) + racMock := configsource.NewRACMock(t, &putEvent) + + cfg := config.Config{AutoConfig: config.AutoConfigConfig{Key: testAutoConfKey}} + cfg.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(racMock.URL) + // A short cleanup interval so the grace expiry is reaped promptly once it passes. + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + relay, err := newRelayInternal(cfg, relayInternalOptions{ + loggers: mockLog.Loggers, + clientFactory: testclient.CreateDummyClient, + }) + require.NoError(t, err) + defer relay.Close() + + h := relayTestHelper{t: t, relay: relay} + env := h.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + // Connect a downstream SDK on the current anchor before the rotation, so this genuinely exercises + // demoting the anchor out from under a live connection. + req := sharedtest.BuildRequestWithAuth("GET", "/all", anchorSDKKey, nil) + sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) + + // Default-rotation patch: promote rotatedAnchorSDKKey to anchor (permanent), demote the old + // anchor with a short grace expiry. The grace window is comfortably longer than the stays-open + // check below so the survival assertion is not racing the expiry. + grace := time.Now().Add(500 * time.Millisecond) + racMock.Send(configsource.MakeAutoConfigPatchEvent( + defaultRotationRep(rotatedAnchorSDKKey, anchorSDKKey, grace.UnixMilli(), 2))) + + // The swap takes effect: the promoted key becomes the environment's anchor. + require.Eventually(t, func() bool { return env.GetAcceptedKeys().Anchor == rotatedAnchorSDKKey }, + 5*time.Second, 5*time.Millisecond, "rotation did not re-anchor to the promoted key") + + // During the grace window the demoted old anchor still authenticates and its stream is undisturbed + // by the swap (a duplicate put from the new anchor's store re-init is fine; only a close fails here). + h.assertSDKEndpointsAvailability(true, anchorSDKKey, "", "") + assertStreamStaysOpen(t, eventCh, 150*time.Millisecond) + + // Once the grace expiry passes, the cleanup ticker drops the old anchor and disconnects its stream. + awaitStreamClosed(t, eventCh, 5*time.Second) + }) + + // After the grace window: the old anchor is gone; the new anchor still authenticates. + awaitCredentialRemoved(t, relay, anchorSDKKey) + h.assertSDKEndpointsAvailability(false, anchorSDKKey, "", "") + h.assertSDKEndpointsAvailability(true, rotatedAnchorSDKKey, anchorMobileKey, multiKeyEnvID) + + // Sanity: the new anchor really is the one relay resolves for upstream. + _, errNew := relay.getEnvironment(sdkauth.New(rotatedAnchorSDKKey)) + assert.NoError(t, errNew) +} diff --git a/relay/endpoints_status_test.go b/relay/endpoints_status_test.go index 9527e4cb..f6371761 100644 --- a/relay/endpoints_status_test.go +++ b/relay/endpoints_status_test.go @@ -15,6 +15,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/internal/util" ct "github.com/launchdarkly/go-configtypes" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" "github.com/launchdarkly/go-sdk-common/v3/ldtime" "github.com/launchdarkly/go-sdk-common/v3/ldvalue" ld "github.com/launchdarkly/go-server-sdk/v7" @@ -248,6 +249,97 @@ func TestEndpointsStatusExpiringSDKKey(t *testing.T) { }) } +// TestEndpointsStatusDuringInFlightRotation drives the real /status handler while an SDK-key re-anchor +// is in flight: the new anchor's client build is wedged via a gated client factory, so the rotation has +// been reconciled but not yet committed. The status request must complete with 200, report the +// PRE-rotation anchor (the rotator has not flipped its anchor pointer until the build is committed), and +// expose a self-consistent accepted set (the reported anchor is present in sdkKeys[]). Once the build is +// released and the rotation commits, /status reports the new anchor. Run under -race to catch any +// unsynchronized read between the status handler and the concurrent re-anchor. +func TestEndpointsStatusDuringInFlightRotation(t *testing.T) { + const newAnchor = c.SDKKey("sdk-status-rotation-new-anchor") + + var config c.Config + config.Environment = st.MakeEnvConfigs(st.EnvMain) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + // Gated factory: healthy for every key except newAnchor, whose build wedges until released. This + // holds the re-anchor mid-flight (pre-commit) so /status observes the pre-rotation anchor. + buildEntered := make(chan struct{}, 1) + release := make(chan struct{}) + inner := testclient.FakeLDClientFactory(true) + gated := func(sdkKey c.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if sdkKey == newAnchor { + buildEntered <- struct{}{} + <-release + } + return inner(sdkKey, cfg, timeout) + } + + relay, err := newRelayInternal(config, relayInternalOptions{ + loggers: mockLog.Loggers, + clientFactory: gated, + }) + require.NoError(t, err) + defer relay.Close() + require.NoError(t, relay.waitForAllClients(time.Second)) + + env, err := relay.getEnvironment(sdkauth.New(st.EnvMain.Config.SDKKey)) + require.NoError(t, err) + require.NotNil(t, env) + + anchor := st.EnvMain.Config.SDKKey + graceExpiry := time.Now().Add(time.Hour) + set, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: newAnchor}). + WithSDKKey(credential.SDKKeyParams{Value: anchor, Expiry: util.PtrOrNil(graceExpiry)}). + Build() + require.NoError(t, err) + + // Drive the re-anchor on a background goroutine; it blocks in the gated build, holding the rotation + // mid-flight (pre-commit). + reconcileDone := make(chan struct{}) + go func() { + defer close(reconcileDone) + env.ReconcileCredentials(set) + }() + <-buildEntered // the new anchor's build is wedged: the rotation is in flight, not yet committed. + + // /status must complete and report the pre-rotation anchor while the rotation is in flight. + r, _ := http.NewRequest("GET", "http://localhost/status", nil) + result, body := st.DoRequest(r, relay) + assert.Equal(t, http.StatusOK, result.StatusCode) + envStatus := ldvalue.Parse(body).GetByKey("environments").GetByKey(st.EnvMain.Name) + + // The scalar anchor is still the pre-rotation key (the anchor pointer flips only on commit). + assert.Equal(t, sdks.ObscureKey(string(anchor)), envStatus.GetByKey("sdkKey").StringValue(), + "status reports the pre-rotation anchor while the rotation is mid-flight") + + // The arrays are self-consistent: the reported anchor is present in sdkKeys[]. + sdkKeys := envStatus.GetByKey("sdkKeys") + require.False(t, findKeyStatusByValue(sdkKeys, sdks.ObscureKey(string(anchor))).IsNull(), + "the reported anchor must be present in sdkKeys[]") + + // The env still serves the previous anchor's client, so it reports connected. + assert.Equal(t, "connected", envStatus.GetByKey("status").StringValue()) + + // Release the build; the rotation commits. + close(release) + <-reconcileDone + + // After the commit, /status reports the new anchor, still present in a consistent sdkKeys[]. + r2, _ := http.NewRequest("GET", "http://localhost/status", nil) + result2, body2 := st.DoRequest(r2, relay) + assert.Equal(t, http.StatusOK, result2.StatusCode) + envStatus2 := ldvalue.Parse(body2).GetByKey("environments").GetByKey(st.EnvMain.Name) + assert.Equal(t, sdks.ObscureKey(string(newAnchor)), envStatus2.GetByKey("sdkKey").StringValue(), + "after the commit, status reports the new anchor") + require.False(t, findKeyStatusByValue(envStatus2.GetByKey("sdkKeys"), sdks.ObscureKey(string(newAnchor))).IsNull(), + "the new anchor must be present in sdkKeys[] after the commit") +} + // TestKeyStatus verifies the helper that converts an accepted key into its status-endpoint JSON form. func TestKeyStatus(t *testing.T) { strptr := func(s string) *string { return &s } From a5fe46bca361242fb4a2dfe7ceffbba5a863dc35 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Fri, 24 Jul 2026 14:24:21 -0700 Subject: [PATCH 51/66] chore(concurrent-keys): remove project-specific details before releasing (#775) Pre-v8 cleanup that removes concurrent-keys project scaffolding while keeping shipped feature code and regression tests. --- .../phase1-T0-reanchor-poc-findings.md | 182 ----- .agent-docs/concurrent-keys/phase1-design.md | 464 ------------ .agent-docs/concurrent-keys/phase1-plan.md | 557 --------------- internal/autoconfig/stream_manager.go | 4 +- .../autoconfig/stream_manager_errors_test.go | 2 +- .../reload_restart_redis_test.go | 2 +- .../env_context_handler_fanout_test.go | 2 +- .../env_context_reanchor_bigsegment_test.go | 8 +- .../env_context_reanchor_helpers_test.go | 116 +++ .../relayenv/env_context_reanchor_test.go | 659 ------------------ .../store_handover_realclient_test.go | 6 +- .../configsource/archive_fixture.go | 218 ------ .../sharedtest/configsource/package_info.go | 4 +- internal/sharedtest/configsource/rac_mock.go | 2 +- internal/store/relay_feature_store.go | 4 +- .../store/store_rebuild_after_close_test.go | 2 +- internal/store/store_refcount_test.go | 2 +- relay/concurrent_keys_auth_test.go | 4 +- relay/concurrent_keys_defensive_test.go | 2 +- relay/concurrent_keys_harness_ref_test.go | 142 ---- 20 files changed, 138 insertions(+), 2244 deletions(-) delete mode 100644 .agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md delete mode 100644 .agent-docs/concurrent-keys/phase1-design.md delete mode 100644 .agent-docs/concurrent-keys/phase1-plan.md create mode 100644 internal/relayenv/env_context_reanchor_helpers_test.go delete mode 100644 internal/relayenv/env_context_reanchor_test.go delete mode 100644 internal/sharedtest/configsource/archive_fixture.go delete mode 100644 relay/concurrent_keys_harness_ref_test.go diff --git a/.agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md b/.agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md deleted file mode 100644 index e27c5e6f..00000000 --- a/.agent-docs/concurrent-keys/phase1-T0-reanchor-poc-findings.md +++ /dev/null @@ -1,182 +0,0 @@ -# T0 — Re-anchoring PoC: Findings - -**Ticket**: [SDK-2530](https://launchdarkly.atlassian.net/browse/SDK-2530) -**Epic**: [SDK-2453](https://launchdarkly.atlassian.net/browse/SDK-2453) -**Design**: [`phase1-design.md`](./phase1-design.md) §7 "Re-anchoring" -**Tests**: [`internal/relayenv/env_context_reanchor_test.go`](../../internal/relayenv/env_context_reanchor_test.go) - -## Purpose - -Validate the upstream SDK-client swap mechanism that **T2.c** will implement, answering the seven -hypotheses from design §7 with durable tests *before* T2 begins. T0 validates feasibility; T2.c -implements. The tests in `env_context_reanchor_test.go` are written against today's primitives so they -survive into T2 as regression tests and as the executable spec for the swap. - -There is no dedicated re-anchor method yet. The closest existing code path is `UpdateCredential` with a -grace period (rotate the primary SDK key, stand up a new client, keep the old one alive during a grace -window). Several PoC tests drive that path and observe where it falls short of the §7 requirements; -those gaps are the concrete acceptance criteria for T2.c. - -## Headline conclusion - -**Re-anchoring is feasible, but it is _not_ a transparent side-effect of today's code — three concrete -gaps must be closed by T2.c/T2.d, and one design assumption (the "shared store") is only true for -persistent stores.** None of the gaps are blockers; each has a clear remedy. The single highest-risk -item is the in-memory data store being rebuilt (emptied) when the new anchor client starts; the remedy is -to hand the existing store over to the new client rather than rebuild it (H5). - ---- - -## Findings per hypothesis - -### H1 — Two SDK clients sharing a `storeAdapter` don't corrupt store invariants - -**Answer: No corruption, but the "shared store" is a misconception for the in-memory store.** - -`SSERelayDataStoreAdapter.Build()` is called once per SDK-client creation (the SDK invokes -`DataStore.Build()` during client init). Each call constructs a **new** `streamUpdatesStoreWrapper` -around a **freshly built** underlying store and atomically swaps `adapter.store` to point at it. So: - -- With the **default in-memory** store, the second client (the new anchor) gets a brand-new, empty, - uninitialized store. Design §7's "two SDK clients … can feed the same store as a side-effect" does - **not** hold here — the new client must re-sync from scratch. -- With a **persistent** store (Redis/DynamoDB), `wrappedFactory.Build()` returns a handle to the same - external database, so the data (and `IsInitialized()`) survive the swap. This is the only - configuration in which the §7 assumption is literally true. - -No invariant corruption occurs in either case (the swap is atomic under the adapter's lock), but the -emptiness of the new in-memory store is the crux of H5. The remedy — handing the existing store over to -the new client rather than rebuilding — is covered under H5. - -### H2 — Downstream SSE connections tolerate the swap - -**Answer: Yes — open connections survive; expect one duplicate `put`.** - -- **Connection survival:** downstream streams live in `envStreams`, keyed by `ScopedCredential`, - entirely independent of the upstream `clients` map. A re-anchor touches only `clients`, the rotator - anchor pointer, and the data store. An open client-side connection (keyed on env ID) keeps receiving - events across the swap (verified live: a `ping` still arrives after re-anchor). Connections are torn - down **only** for credentials that are actually removed (`removeCredential` → `RemoveCredential` → - `Close()`), which is the intended graceful-rotation behavior, not a swap side-effect. -- **Duplicate `put`:** the new anchor client's initial sync calls `store.Init(allData)`, which flows - through the store wrapper → `SendAllDataUpdate` → re-broadcast of a full `put`/`ping` to every - connected downstream stream. From a downstream SDK's perspective this is a duplicate put. It is - tolerable (SDKs apply puts idempotently) but **T2.c must expect it**; it is not corruption. - -### H3 — Big-segment sync after re-anchor - -**Answer: Re-wiring is required. It is NOT handled today.** - -`bigSegmentSync` is constructed once in `NewEnvContext`, wired to `envConfig.SDKKey` and `envConfig.EnvID` -at construction. The PoC confirms the swap path neither recreates the synchronizer nor informs it of the -new key (the `BigSegmentSynchronizer` interface has Start / HasSynced / SegmentUpdatesCh / Close — **no -credential-replacement method**). After a re-anchor it keeps polling/streaming big-segment data on the -**old** anchor key, which will break once the old key is revoked. - -**T2.d action:** add a re-wire path to `BigSegmentSynchronizer` (a `ReplaceCredential`-style method, -mirroring the event dispatcher / metrics publisher) **or** recreate the synchronizer on each re-anchor. -The "recreate" option is simpler; the "re-wire" option avoids dropping in-flight sync state. - -### H4 — `httpconfig` stays functional after re-anchor - -**Answer: Yes — no re-wire needed.** - -`httpconfig` carries TLS / proxy / transport / user-agent configuration plus the SDK key, but the only -key-dependent artifact is the `Authorization` default header on the pre-built `SDKHTTPConfig`. Relay -injects the *builder* (`SDKHTTPConfigFactory`), not the pre-built config, into `ld.Config.HTTP`, and the -SDK rebuilds the HTTP config with the new anchor key when it constructs the new client — so the -`Authorization` header is set correctly for the new anchor automatically. The pre-built `SDKHTTPConfig` / -`Client()` (used for event + big-segment transport) is key-independent except for that header, and those -components set their own auth per request rather than reading it from `httpconfig`. No action required. - -### H5 — Order of operations (start-new → swap pointer → close-old) - -**Answer: The recommended order is necessary but NOT sufficient for the in-memory store.** - -Because building the new client is what rebuilds (and empties) the in-memory store (H1), there is a -window after the swap in which evaluations see an empty store until the new anchor finishes its initial -sync — *regardless* of operation order. The PoC shows the env's store is replaced with a fresh, -uninitialized store as soon as the new client is registered. - -**Recommended remedy: hand the existing store over to the new client.** Because relay owns the store -implementation (it hands the SDK a single `storeAdapter`), the re-anchor can reuse the existing store for -the new client instead of letting `Build()` construct a fresh one — concretely, make -`SSERelayDataStoreAdapter.Build()` return its existing store when one is already present (or otherwise -seed the new client with the old client's store). The new anchor then reads populated, initialized data -immediately, so there is no empty-store window. Validated by -`TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow`. - -This is simpler than the alternatives originally considered — gating the swap on `Initialized()`, -mandating a persistent store, or otherwise decoupling store from client — and supersedes them. - -**Store-lifecycle caveat:** `streamUpdatesStoreWrapper.Close()` closes the underlying store. With -handover the retiring and new clients share one underlying store, so closing the retiring client must -**not** close it — the adapter (not the client) must own the store's lifecycle. This is not reproducible -with the fake client used in the PoC; verify against the real client in T2.c. - -**Recommended order, refined:** start new client (handing over the existing store) → swap anchor pointer -and re-wire peripherals → close old client, ensuring that close does not tear down the shared store. - -### H6 — Behavior during the swap window (requests arriving mid-swap) - -**Answer: Today there is a gap — `GetClient()` returns nil mid-swap.** - -`GetClient()` returns `clients[keyRotator.SDKKey()]`. In the current path the rotator's primary key flips -to the new key **synchronously** inside `UpdateCredential`, but the new client is created on a background -goroutine (`go startSDKClient`) and registered only afterward. The PoC deterministically observes -`GetClient() == nil` in that window (gated client factory, no sleeps). A request arriving mid-swap gets a -nil client. - -**T2.c action:** do not advance the anchor pointer until the new client is registered (and ideally -`Initialized()`). Combined with H5/H7, the rule is: **construct + initialize the new client first, then -atomically flip the anchor pointer.** - -### H7 — Failure mode: new client init fails - -**Answer: Today a failed re-anchor breaks the environment. Atomicity/rollback is required.** - -When the new anchor client fails to initialize, the rotator has already flipped the anchor pointer to the -new key, but no client exists for it — so `GetClient()` returns nil **even though the old anchor's client -is still alive and valid** during its grace period. The PoC confirms `GetInitError()` is set, `GetClient()` -is nil, and the old client is still present in the `clients` map (so the data path *could* have been -preserved). - -**T2.c action (this is §8's atomicity requirement):** validate that the new client initializes **before** -swapping the anchor pointer; on failure, roll back to the old anchor and preserve the previous accepted -set. Log a structured error and alarm (per §9). - ---- - -## Consolidated requirements for T2.c / T2.d - -| # | Requirement | From | -|---|---|---| -| 1 | Construct + initialize the new anchor client **before** flipping the anchor pointer; flip atomically. | H5, H6 | -| 2 | On new-client init failure, roll back to the old anchor; preserve previous accepted set; log + alarm. | H7 | -| 3 | Hand the existing store over to the new client (make `SSERelayDataStoreAdapter.Build` reuse its store) so there is no empty-store window; ensure the retiring client's `Close()` does not tear down the shared store. | H1, H5 | -| 4 | Re-wire big-segment sync on re-anchor (add a replace-credential method, or recreate the synchronizer). | H3 | -| 5 | Continue calling `ReplaceCredential` on the event dispatcher + metrics publisher (already wired in `addCredential`). | §7 table | -| 6 | Expect a duplicate downstream `put` from the new anchor's initial sync; ensure downstream connections are not torn down for retained credentials. | H2 | -| 7 | No `httpconfig` change needed. | H4 | - -## What did NOT need changing - -- `httpconfig` (H4). -- Downstream SSE routing / `envStreams` (H2) — already credential-scoped and independent of the anchor. -- Event dispatcher + metrics publisher already expose `ReplaceCredential` and are already called from - `addCredential` on an SDK-key change. - -## Test inventory - -All tests are in [`internal/relayenv/env_context_reanchor_test.go`](../../internal/relayenv/env_context_reanchor_test.go), -prefixed `TestReanchorPoC_H_…`: - -- `H1_SharedStoreAdapterRebuildSemantics` — in-memory rebuild vs. persistent-store preservation. -- `H2_DownstreamConnectionSurvivesReAnchor` — live client-side connection survives the swap. -- `H2_NewClientInitialSyncRebroadcastsPut` — duplicate `put` is produced and counted. -- `H3_BigSegmentSyncIsNotReWiredOnReAnchor` — synchronizer keeps the old key; no re-wire today. -- `H4_HTTPConfigIsKeyIndependentExceptAuthHeader` — only the auth header is key-dependent. -- `H5_InMemoryStoreIsWipedByReAnchor` — store replaced/empty after swap. -- `H5_StoreHandoverAvoidsEmptyWindow` — reusing the store across the swap avoids the empty window (the remedy). -- `H6_AnchorPointerFlipsBeforeNewClientIsRegistered` — `GetClient()` nil mid-swap (deterministic). -- `H7_FailedNewClientLeavesEnvWithoutAnchorClient` — failed swap breaks the env; old client still alive. diff --git a/.agent-docs/concurrent-keys/phase1-design.md b/.agent-docs/concurrent-keys/phase1-design.md deleted file mode 100644 index 6e36bf17..00000000 --- a/.agent-docs/concurrent-keys/phase1-design.md +++ /dev/null @@ -1,464 +0,0 @@ -# Phase 1 — Concurrent SDK Keys in Relay Proxy: Design - -**Epic**: [SDK-2453](https://launchdarkly.atlassian.net/browse/SDK-2453) -**Backend tech spec**: [Confluence 4186243250](https://launchdarkly.atlassian.net/wiki/spaces/PD/pages/4186243250/Tech+Spec+Concurrent+SDK+Keys) -**Companion**: [`phase1-plan.md`](./phase1-plan.md) (tasks, sequencing, estimates) - -This document is the canonical reference for the *what* and *why* of Phase 1. The companion plan covers *how*. Agents working on individual tasks should read both. - ---- - -## 1. Overview - -LaunchDarkly is rolling out **concurrent SDK keys** — the ability for a single environment to have *multiple* SDK keys and *multiple* mobile keys simultaneously. Phase 1 brings this capability to the Relay Proxy. - -### Why - -Customers (notably Block/Square, Confluent) maintain dozens or hundreds of services that share the same LaunchDarkly environment. Today each service uses the same single SDK key per environment. If that key is compromised, customers face large-scale operational toil rotating it across every service. - -Concurrent keys let customers issue distinct keys per service, reducing blast radius and supporting independent key lifecycles. - -### Scope - -| In scope (Phase 1) | Out of scope | -|---|---| -| Multiple SDK keys per environment | Multiple client-side IDs per environment (deferred to a later project) | -| Multiple mobile keys per environment | Views / payload filtering V2 (Phase 2 — mega stream) | -| Per-key expiry and graceful rotation | Per-key event attribution (analytics events collapse to the env's anchor key) | -| Delivery via **Relay Auto Config (RAC)** | Manual config (TOML / env vars) — stays single-key in Phase 1 (lifted in Phase 2) | -| Delivery via **offline-mode archive** | | -| Implementation in Relay v8, merged forward to Relay v9 | | - -### The "trusted source" restriction - -Relay authenticates *upstream* with only one SDK key per environment — the **anchor**. Additional keys are accepted locally but not verified upstream. That makes "trust the source of additional keys" load-bearing for safety. - -LaunchDarkly-generated sources (RAC, offline archive) are trusted: they only ever carry an environment's real keys, so a wrong-environment key can't appear. Hand-entered manual config is *not* trusted — a typo could silently leak this env's data to an SDK using a wrong-env key and misattribute its events to this env. Phase 1 therefore accepts additional keys *only* from RAC and offline archives. - -Manual config support returns in Phase 2 via the mega stream, which verifies every key individually. - -Customer impact: ~50% of relay customers use RAC and benefit immediately. The other ~50% (manual config) wait for Phase 2. The team has reviewed this trade-off — see [Confluence 4979425298](https://launchdarkly.atlassian.net/wiki/spaces/PD/pages/4979425298/Relay+Proxy+Auto+Config+Risk+Assessment). - ---- - -## 2. Preamble: How Relay Auto Config (RAC) works - -The §2.4 event table is unreadable without this context. - -**RAC is a push channel from LaunchDarkly *to* relay.** It's a single long-lived SSE stream over HTTPS. Relay opens it at startup using a special "relay token" (distinct from any SDK key) and consumes the stream's messages. - -**Lifecycle**: -1. Relay starts up. If RAC is configured, relay opens an SSE connection to LaunchDarkly's RAC endpoint. -2. LaunchDarkly responds with an initial `put` message containing the full state of every environment this relay should know about. Relay creates an `EnvContext` per env and opens upstream SDK clients on the anchor key. -3. While the SSE stream is open, LaunchDarkly pushes incremental messages: `patch /environments/$ENVID` (state changed), `delete /environments/$ENVID` (removed), `put /` (full refresh — rare, usually on reconnect). -4. Connection drops → relay reconnects with backoff and reconciles against the next `put`. - -The path notation (`/environments/$ENVID`) is a **JSON path within the RAC document**, not an HTTP route. It identifies which part of relay's internal state the message addresses. Think JSON Patch semantics. - -**Offline mode is the same in shape, different in transport.** No stream. LaunchDarkly tooling generates an archive file with the same `EnvironmentRep` shape. Relay reads it on startup and reconciles on reload. - -**Don't confuse RAC with the SDK streaming endpoint.** Relay has two kinds of upstream connection: RAC (one per relay, carries config) and the SDK stream (one per env, anchored by SDK key, carries flag/segment data). Phase 1 changes the SDK-stream-per-env story; RAC itself is unchanged in transport. - -## 3. Preamble: RAC vs Manual Config - -Relays use *either* manual config *or* RAC — not both. They're alternative top-level configuration approaches. - -- **Manual config**: TOML file or `LD_*` env vars list each environment explicitly with its SDK key, mobile key, and env ID. Static — operator edits the file and reloads. -- **RAC**: TOML file has a single `[AutoConfig]` block with a relay token. LaunchDarkly streams the environment list. No `[Environment ...]` blocks needed. - -A relay instance is one or the other, decided at deployment time. - -RAC is **enterprise-only** in LaunchDarkly's pricing tiers. Lower-tier customers physically cannot use RAC and use manual config by necessity. This pricing reality is why the trusted-source restriction (§1) excludes ~50% of relay customers from Phase 1. - ---- - -## 4. Architectural pillars - -Phase 1 rests on three commitments. The whole design is consistent with these; they hold across every code path and test. - -### 4.1 One upstream connection per environment, on the anchor - -Relay opens exactly one upstream SDK client per env, authenticated with the anchor SDK key. All other accepted keys (server and mobile) are matched locally against the request's `Authorization` header and served off the same data store. They never open their own upstream connection. - -This generalizes existing behavior: today's mobile keys and client-side IDs already behave this way (verified in [`internal/relayenv/env_context_impl.go`](../../internal/relayenv/env_context_impl.go); only `config.SDKKey` calls `startSDKClient`). Phase 1 extends "local match only, no upstream client" to non-anchor *server* keys. - -**Trade-off**: this is a deliberate choice over the alternative (one upstream client per accepted key, the approach in Matthew Keeler's PoC at PR #675). Reasons we chose single-anchor: -- **Connection-count efficiency** — a customer with 50 keys × 10 envs × 10 filters would otherwise hold thousands of upstream streams. -- **Phase 2 alignment** — Phase 2's mega stream is one connection per environment. Single-anchor is closer to that target. - -The cost is re-anchoring complexity (§7). We accept that cost. - -### 4.2 The anchor - -The anchor is **the SDK key the singular `sdkKey.value` field points to**, identified by byte-equality against an entry in the `sdkKeys[]` array. There is no `isDefault` flag in the wire format — the value match *is* the signal. - -The backend designates the anchor; relay is passive. Relay reads `sdkKey.value` and uses that key for upstream. - -**Invariants** (maintained by the backend): -- `sdkKey` always names a non-expiring key. -- The backend blocks deleting or expiring the last non-expiring key in an environment. On default rotation, the backend promotes another non-expiring key first, then flips `sdkKey.value`. -- The new anchor's entry in `sdkKeys[]` continues to carry no `expiry`. The old anchor (now demoted) carries an `expiry`. - -**Re-anchor trigger**: whenever `sdkKey.value` changes. This is the single trigger for an upstream-client swap. See §7 for the mechanism. - -**Mobile-key analog**: the singular `mobKey` field is the default mobile key for events. No upstream connection — mobile keys are local-match-only — but `mobKey` plays the same back-compat singular-pointer role. - -### 4.3 Events collapse to anchor per kind - -Analytics events forward upstream under the env's anchor key of each kind. Two dispatchers per env: one for SDK events under `sdkKey.value`, one for mobile events under `mobKey`. The dispatcher uses its stored `authKey`, not the credential on the incoming request. - -**Why no per-key event attribution**: SDK keys are *secrets* and not appropriate as metric/analytics tags. LaunchDarkly provides customer-facing tagging mechanisms (context attributes, environment tags) for slicing events. The trusted-source restriction makes anchor attribution safe — every accepted key truly belongs to this env, so anchor attribution lands on the right env. We lose per-key granularity, not env correctness. - -**Asymmetry — diagnostic events**: diagnostic events (SDK self-reported initialization, errors) take a different code path. They proxy the incoming request's headers verbatim, including the Authorization header carrying the original credential. This preserves operational debug value — *which* SDK reported this — at the cost of asymmetry with analytics. We accept this. (Long-term, a metadata-header approach could provide symmetric attribution; out of scope for Phase 1.) - ---- - -## 5. Wire format - -Both RAC and the offline archive carry the same `EnvironmentRep`. One parsing change covers both sources. Producers already emit this format — relay can implement and test against captured payloads today. - -### Example RAC `event:put` - -```json -{ - "path": "/", - "data": { - "environments": { - "68e5179e8307e4099c277e2a": { - "envId": "68e5179e8307e4099c277e2a", - "envKey": "production", - "envName": "Production", - "projKey": "...", - "projName": "...", - "secureMode": false, - "version": 26, - "sdkKey": { "value": "sdk-9409..." }, - "mobKey": "mob-f41c...", - "sdkKeys": [ - { "key": "new-production-default", "value": "sdk-9409..." }, - { "key": "another-one", "value": "sdk-38b0..." } - ], - "mobileKeys": [ - { "key": "mob-key-50bca22351", "value": "mob-f41c..." } - ] - } - } - } -} -``` - -The offline archive wraps the same `env` object per entry: `{"env": ..., "dataId": "..."}`. - -### Shape rules - -- Array entries: `{ "key": , "value": , "expiry"?: }`. -- Singular `sdkKey` is an **object** (`{"value": ...}`); may carry the legacy `expiring{value, timestamp}` slot during default rotation. -- Singular `mobKey` is a **plain string**. Shape asymmetry is historical (mobile keys never had a legacy expiring slot). -- Anchor = `sdkKeys[]` entry whose `value` matches `sdkKey.value`. No `isDefault` flag — value match is the signal. -- Arrays are *inclusive* of the default — the anchor entry is *in* `sdkKeys[]`, not separate. -- `expiry` is present only while a key is expiring; omitted otherwise; never null. -- The legacy `sdkKey.expiring{}` slot is populated **only during default rotation** (old-relay back-compat). Non-default key expiring uses only the array `expiry`. -- Old relays ignore unknown JSON fields and continue using singular `sdkKey`/`mobKey` — additive, fully backward-compatible. - -### Terminology - -Aligned with the backend tech spec's `accounts.sdk_keys` table: - -- **`name`** = display name (e.g. "Default SDK Key"). Used in the UI. **Not in the wire format** — relay doesn't need it. -- **`key`** = identifier (e.g. "default-sdk"). Non-secret. Carried in wire as `key`. -- **`value`** = the credential secret (e.g. `sdk-xxxx-...`). Carried in wire as `value`. - -**Naming trap in code**: relay's existing types `SDKKey`, `MobileKey`, `SDKCredential` refer to what the wire format calls `value`. The wire's `key` field is the *identifier*, a different thing. Do not rename the existing relay types — they're stable — but call out the trap in code comments. - -A canonical comment for the wire-type definition (subject to bikeshed at PR time): - -```go -// EnvironmentRep carries an environment's wire shape from RAC and the offline -// archive (same struct serves both — keep them aligned). -// -// FIELD NAMING — read this before changing anything: -// -// sdkKey is the singular *default* SDK key for the environment. It's an -// object ({"value": "sdk-..."}) so it can also carry the legacy -// sdkKey.expiring{value, timestamp} slot during default rotation -// (back-compat for relays predating concurrent keys). -// -// mobKey is the singular default mobile key. It's a *plain string* -// because mobile keys never had a legacy expiring slot. The shape -// asymmetry is historical, not a design choice. -// -// sdkKeys/mobileKeys are the authoritative full accepted set. Entries: -// { key: , value: , expiry?: } -// -// TERMINOLOGY: -// The wire "key" field is the human-readable IDENTIFIER (e.g. "default-sdk"), -// non-secret. The wire "value" field is the actual CREDENTIAL string (e.g. -// "sdk-xxxx-..."), which is the secret. Note that relay's own types -// (SDKKey, MobileKey, SDKCredential) refer to what the wire calls "value" — -// they're misnamed by today's standards but stable, so do not rename. -// -// Anchor selection: anchor = the sdkKeys entry whose `value` matches -// sdkKey.value. No isDefault flag. See phase1-design.md §4.2. -``` - ---- - -## 6. Credential lifecycle - -### 6.1 Expiry model - -Each entry in `sdkKeys[]` / `mobileKeys[]` carries an optional `expiry` field (Unix-ms timestamp). When present, the key is being phased out — relay drops it when `expiry` passes. When absent, the key is permanent. - -**Two removal paths**: - -- **Graceful**: key has `expiry` set. Relay's existing periodic ticker (`StepTime` → `cleanupExpiredCredentials`) drops the key when the timestamp passes and disconnects downstream SDKs using it. -- **Immediate**: key omitted from the next RAC patch / archive reload. Relay diffs the accepted set on reconcile, finds the missing key, and revokes it now. - -**Edge case**: a key was in graceful state, then omitted entirely → treat as immediate (race-ahead-of-timer). - -### 6.2 Generalize the `Rotator` - -Today's `Rotator` ([`internal/credential/rotator.go`](../../internal/credential/rotator.go)) tracks one primary SDK key + one deprecated-with-expiry slot + single primary mobile key + single primary env ID. Generalize to: a *set* of accepted keys (server + mobile) with optional per-key expiry, plus a designated anchor. - -**Reuse the existing `StepTime` machinery** — generalize from the single `expiring` slot to per-array-key. No new periodic infrastructure. - -**Mobile-key panic**: today, `Rotator.RotateWithGrace(MobileKey, gracePeriod)` panics with `"programmer error: mobile keys do not support deprecation"`. The panic is a guard against an unsupported API state, not a safeguard against a hazard — there was no data-model slot for an expiring mobile key, so the code failed loud rather than store junk. Phase 1's data-model generalization provides the slot; the panic guard is removed alongside. - -### 6.3 Legacy `sdkKey.expiring{}` back-compat - -On default rotation the backend mirrors expiry info into both: -- The old default's entry in `sdkKeys[]` gets `expiry: ` (new field). -- The legacy `sdkKey.expiring{value, timestamp}` slot gets the same (old field, for old relays). - -**Decision**: new relays trust the array. The legacy `sdkKey.expiring{}` field is treated as a write-only back-compat shim — new relays do not read it. (Working assumption pending team confirmation.) - -**Precision (as shipped):** "do not read it" applies when the arrays are present. For an *old-format* payload (no `sdkKeys[]`), the legacy slot is the only source for the deprecated key, and relay does read it there — synthesizing it into the accepted set with its expiry. The rule as implemented: arrays present ⇒ legacy slot ignored; arrays absent ⇒ legacy slot honored. - ---- - -## 7. Re-anchoring - -When `sdkKey.value` changes (voluntary rotation *or* current default expiring and being replaced by a promoted non-expiring key), relay must swap its upstream client to the new anchor while preserving downstream SDK connections. - -This is the highest-risk piece of Phase 1. The **T0 PoC** validated the swap mechanism against seven hypotheses; the durable tests live in `internal/relayenv/env_context_reanchor_test.go` and the per-hypothesis findings are in [`phase1-T0-reanchor-poc-findings.md`](./phase1-T0-reanchor-poc-findings.md). The findings are summarized below as the spec for T2.c / T2.d. - -### Required order of operations - -``` -1. Build the new anchor's SDK client, handing over the existing data store (do not flip the anchor pointer yet). -2. Wait for the new client to report Initialized() == true. -3. Atomically flip the rotator's anchor pointer. -4. Call ReplaceCredential on event dispatcher + metrics publisher. -5. Re-wire (or recreate) big-segment sync. -6. Close the old anchor's client (after its grace period elapses for downstream traffic), ensuring its Close() does not tear down the now-shared store. -``` - -This order is *necessary* — the PoC found that flipping the pointer too early leaves `GetClient()` nil mid-swap (H6) and breaks the env on init failure (H7) — and *sufficient* only with the store-handling approach below. - -> **Implementation notes — as shipped (recorded at final review, 2026-07-18):** -> -> - **Step 6 as implemented:** the demoted anchor's *upstream client* is closed at commit time, not after the grace period. With store handover, a second live upstream client would double-broadcast every update into the shared store wrapper. The demoted key's *credential mappings* stay registered until its grace expires, so downstream SDKs authenticating with it keep working through the window — only the upstream connection goes early (`removeCredential` later finds no client to close). -> - **Case B in practice:** because the demoted client is closed at commit, a re-promoted former anchor has no live client and takes the build path. The "reuse a live client" branch survives defensively; `NewAnchorPreviouslyAccepted` governs credential-mapping registration only. -> - **Operational characteristic:** the synchronous client build runs on the shared RAC dispatch goroutine. While one environment re-anchors, other environments' config updates (and the expiry tickers, which share the reconcile lock) wait — up to `InitTimeout` (default 10s) per re-anchoring environment in the worst case. Data-plane serving is unaffected throughout. This is the accepted cost of build-before-flip; a mass rotation across N environments serializes to roughly N × build time on the config plane. - -### Two re-anchor cases — Case A (new key) vs Case B (already-accepted key) - -The sequence above is **Case A**: the new anchor is a key relay has not previously accepted, so no SDK client exists for it. Relay must build one, hand over the store, wait for `Initialized()`, then flip and re-wire. - -**Case B** is the abbreviated path taken when the new anchor's key **already has a live SDK client** — most commonly a *former* anchor that is still inside its grace period (it was demoted on an earlier rotation, kept alive to serve downstream traffic, and is now being promoted back). In that situation: - -1. **No `Build`.** The existing client is reused as-is — there is no second upstream connection to stand up. -2. **No store handover.** The store the existing client created is already populated and initialized; nothing is handed over because nothing new is constructed. -3. **Atomically flip the rotator's anchor pointer** (`CommitAnchor`) — identical to Case A step 3. -4. **Call `ReplaceCredential`** on the event dispatcher + metrics publisher — identical to Case A step 4. -5. Big-segment sync is re-wired (T2.d), identical to Case A. -6. The retiring anchor's client is closed by the existing `removeCredential` path when its own grace period ends — identical to Case A step 6. - -The two paths **converge after the flip**: steps 3–6 are the same. The only difference is the front of the sequence — Case A builds + initializes + hands over the store; Case B reuses what is already there and does none of that. The caller branches on whether a client already exists for the new anchor (`c.clients[newAnchor] != nil`). - -Because Case B does no client build, there is no init-failure rollback to consider for it — the client it reuses was already initialized and serving. Rollback handling (preserve previous anchor, log a structured error) applies to **Case A only**. - -**Reconcile/additions interaction:** so the synchronous re-anchor owns the new anchor's setup end-to-end, `Rotator.Reconcile` does not flip the anchor itself — it returns a `ReconcileResult.AnchorChange` and the caller invokes `CommitAnchor` at the right moment. In **Case A** the new anchor would otherwise appear in the reconcile's `additions` list and `addCredential` would fire an *async* `startSDKClient` that races the synchronous build — so Reconcile strips the new anchor from `additions` in Case A and the synchronous path installs the peripherals (envStreams, handlers, connection mapping) itself. In **Case B** the new anchor was already accepted, so it was never going to appear in `additions` — no stripping is needed there. - -### The data store: hand the existing store over to the new client - -An earlier version of this design assumed two SDK clients pointed at the same env would feed the *same* data store as a side-effect. The PoC (H1, H5) showed this is **wrong for the in-memory store**: each SDK client construction calls `storeAdapter.Build()`, which atomically swaps in a *new, empty* store, so the new client would otherwise have to re-sync from scratch (an empty-store window). This affects only the in-memory case; with a persistent store (Redis, DynamoDB) the data lives outside the wrapper and survives the swap. - -**Chosen remedy: hand the existing store over to the new client.** Because relay owns the store implementation (it hands the SDK a single `storeAdapter`), the re-anchor reuses the existing store for the new client instead of letting `Build()` construct a fresh one — concretely, make `SSERelayDataStoreAdapter.Build()` return its existing store when one is already present (or otherwise seed the new client with the old client's store). The new anchor then serves populated, initialized data immediately, with no empty-store window. Validated by `TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow`. - -This is the concrete form of decoupling the store's lifecycle from the client's. Two alternatives were considered and rejected as heavier: gating the swap on the new client reaching `Initialized()` (still leaves a window for already-connected reads and keeps the store coupled to client construction), and mandating a persistent store for graceful re-anchor (constrains deployments). - -**Store-lifecycle caveat:** `streamUpdatesStoreWrapper.Close()` closes the underlying store. With handover the retiring and new clients share one underlying store, so closing the retiring client must **not** close it — the adapter (not the client) must own the store's lifecycle. (Not reproducible with the fake client used in the PoC; verify against the real client in T2.c.) - -### Component re-wiring on re-anchor - -| Component | Today | On re-anchor | -|---|---|---| -| Data store | Rebuilt per client by `storeAdapter.Build()` | Hand the existing store over (adapter reuses it); the retiring client's `Close()` must not tear it down (T2.c) | -| Event dispatcher | Has `ReplaceCredential` | Call `ReplaceCredential` (T2.c) | -| Metrics publisher | Has `ReplaceCredential` | Call `ReplaceCredential` (T2.c) | -| Big-segment sync | Wired at construction; no replacement method | Re-wire (add a replace-credential method) **or** recreate the synchronizer (T2.d). Recreate is simpler; re-wire preserves in-flight sync state. | -| `httpconfig` | Built via injected *builder* (not pre-built config) | **No change needed** — the SDK rebuilds with the new anchor key automatically (PoC H4) | -| Downstream SSE connections | Keyed on `ScopedCredential`, independent of anchor | Survive automatically (PoC H2); expect one duplicate `put` from the new client's initial sync | - -### Failure handling - -If the new client fails to initialize, the swap **rolls back**: the rotator's anchor pointer stays on the old key (the caller simply does not call `CommitAnchor`), the anchor-related changes are reverted — the previous anchor is re-admitted and kept serving even if the payload revoked it outright, and a brand-new failed anchor is dropped — and a structured error is logged. The old anchor's client continues to serve. It applies to **Case A only** — Case B reuses an already-initialized client and has nothing to fail. Relay has no dedicated alarm infrastructure today; an `Error`-level structured log (`globalLoggers.Errorf`) is the strongest signal available and is sufficient. - -**As-shipped rollback scope (recorded at final review, 2026-07-18):** the rollback is scoped to the *anchor change*, not the whole reconcile — other credential changes in the same payload (adds and removals of non-anchor keys) stand. **Recovery:** the payload's version was already recorded at the stream parse boundary, so an identical retry — or a reconnect's fresh `put` at the same version — is deduplicated; the environment stays on the previous anchor until the backend sends a *version-bumped* update. - -### Consolidated specification for T2.c / T2.d - -| # | Requirement | Source | Owner | -|---|---|---|---| -| 1 | **Case A**: build + initialize the new anchor client *before* flipping the pointer; flip atomically via `CommitAnchor`. **Case B** (new anchor already has a live client): skip the build, reuse it, then flip. | H5, H6 | T2.c | -| 2 | **Case A** init failure: roll back (do not `CommitAnchor`); preserve previous accepted set; log a structured error. Not applicable to Case B. | H7 | T2.c | -| 3 | **Case A**: hand the existing store over to the new client (adapter reuses its store, refcounted so the retiring client's `Close()` does not tear it down). **Case B**: no handover — the store is already populated. | H1, H5 | T2.c | -| 4 | Re-wire big-segment sync on re-anchor (recreate or replace-credential). Same for both cases. | H3 | T2.d | -| 5 | Call `ReplaceCredential` on event dispatcher + metrics publisher (synchronously, in the re-anchor sequence — not via `addCredential`, since Reconcile strips the new anchor from `additions`). Same for both cases. | §7 | T2.c | -| 6 | Expect duplicate downstream `put`; retain connections for credentials still in the accepted set. | H2 | T2.c (awareness) | -| 7 | No `httpconfig` change. | H4 | n/a | - ---- - -## 8. Processing & lifecycle - -Three event types, two source paths (RAC + offline), one integration point (`ReconcileCredentials` on `EnvContext`). - -| Event | RAC trigger | Offline trigger | Relay action | -|---|---|---|---| -| Env added | `patch /environments/$ENVID` (env not known) | new archive entry on reload | Create env; open upstream connection on anchor (online only); map all accepted keys into lookup | -| Keys change | `patch /environments/$ENVID` (env known, payload differs) | archive update on reload | Reconcile accepted `sdkKeys`/`mobileKeys`; re-anchor if `sdkKey.value` changed | -| Env deleted | `delete /environments/$ENVID` | env removed from reloaded archive | Tear down env + upstream connection + mappings | - -### Order of operations (keys change) - -Within a single `keys change` event, the order is: - -1. **Add new keys** (new credential entries added to accepted set, handlers built). -2. **Re-anchor** if `sdkKey.value` changed (swap upstream client, re-wire peripheral components). -3. **Remove expiring/omitted keys** (drop entries, disconnect downstream SDKs that were using them). - -This order ensures the accepted set is a *superset* during the transition. The new anchor's client comes up before the old anchor's client tears down. Downstream SDKs are never spuriously rejected mid-update. - -### Atomicity - -**As shipped (this resolves the earlier open question):** a **malformed payload** is all-or-nothing — it is rejected at the stream parse boundary before any state mutation, so the previous accepted set is fully preserved (§9). A **re-anchor init failure** rolls back the anchor change only; the payload's other adds and removals stand (§7 failure handling). Full-reconcile rollback was considered and rejected: it would add snapshot/restore machinery across mappings and streams for no clear benefit given trusted sources, and the partial semantics are strictly safer for the non-anchor keys involved (valid new keys start working; revoked keys stay revoked; the anchor never breaks). - -### Edge cases - -- **De-expiry** (key was expiring; new payload omits `expiry`): cancel the scheduled drop. -- **Rename** (same `value`, different `key` identifier): no-op for credential set; only update status-endpoint display. -- **Mixed update** (add + re-anchor + remove in one patch): apply in the order above. - ---- - -## 9. Defensive behavior — malformed payloads - -When relay receives a malformed RAC payload — most importantly, `sdkKey.value` not present in `sdkKeys[]`, or `sdkKey` field missing entirely — the backend invariants of §4.2 have been violated. - -**Decision** (confirmed with the team): - -1. **Preserve the previous accepted set.** Do not apply the malformed update. Log a structured error. Alarm. -2. **Disconnect and reconnect the RAC stream with jitter.** The backend believes the patch was applied — RAC is one-way push and relay has no NAK channel. Without a reconnect the backend won't send a fresh state; it expects relay to be in sync. Reconnecting forces a fresh `put` on the new connection, which gives relay a clean baseline. -3. Do *not* silently fall back to the first entry in `sdkKeys[]` (silent and dangerous). -4. Do *not* leave the env in a half-applied state. - -This is the same atomicity principle as §8, applied at the boundary between trusted-source input and relay's internal state, with the added piece (reconnect) needed because RAC has no acknowledgment mechanism for failed-payload rejection. - -**Implementation notes (recorded at final review, 2026-07-18):** validation runs at the stream parse boundary, *before* the message's version is recorded — this is what makes the forced fresh `put` (which carries the same version) re-processable rather than deduplicated away. The shipped validation is also stricter than the two cases named above: array entries with empty `value`s, and a defined `mobKey` absent from `mobileKeys[]` (the mobile analogue of the anchor invariant), are also rejected as malformed. - ---- - -## 10. Backwards compatibility - -Two assertions: - -1. **Payload is additive.** Relays that don't understand the new `sdkKeys`/`mobileKeys` array fields ignore them and continue using the singular `sdkKey`/`mobKey`. No coordinated upgrade required. -2. **One representation for all relays and both sources.** No per-version fork of `EnvironmentRep`. - -**Bidirectional upgrade compat**: customers can upgrade their backend before relay, or relay before backend, in any order. The slowest party uses singular fields; the faster party emits arrays. Both states converge to single-key behavior until both are upgraded. - -**Verification**: `DisallowUnknownFields` is not used anywhere in relay's env-parse path. The additive claim holds — Go's default JSON decoder silently ignores unknown fields. Confirmed as T3.a pre-work. - -**Downgrade story**: open question for the team. Rolling relay back from Phase 1 to a pre-Phase-1 build means SDKs using non-anchor keys would lose connectivity. Document as a release-note consideration. - ---- - -## 11. Manual configuration - -Manual config (TOML file or `LD_*` env vars) continues to support **exactly one SDK key + one mobile key + one env ID per environment**, as today. The schema doesn't change. Manual-config customers see zero behavior change from Phase 1. - -**The PoC's manual-multi-key additions must not be inherited.** SDK-2415 added `AdditionalSDKKeys` and `LD_ADDITIONAL_SDK_KEYS_*` support. We deliberately *do not* want this. T3 review must verify neither pattern appears in `config/config.go` or `config/config_validation.go`. - ---- - -## 12. Internal model - -``` -Environment - envID - identifiers (key, name, proj…) - anchorKey (the one upstream-auth key) - acceptedKeys: KeySet (server + mobile, local match) - clientSideID (single) - upstreamConnection (one, on the anchor) - dataStore - │ - └─── KeySet - keys (equivalent peers, server or mobile) - per-key optional expiry -``` - -**`KeySet`** generalizes today's `Rotator` ("primary + deprecated-with-expiry") into "set of accepted keys + anchor." - -**Routing/auth is a local lookup**: a connecting credential is matched against the accepted set → the environment → served off the single anchor connection. The env ID registers exactly once. - -**No per-view structure anywhere in Phase 1.** Premature abstraction — Phase 2's mega stream is still speculative. Keep the model flat. A key is just an accepted credential. Don't preemptively add `viewKeys` fields or "scope" abstractions. - ---- - -## 13. Recorded decisions - -| Decision | Rationale | Alternatives rejected | -|---|---|---| -| Trusted sources only (RAC + offline archive) in Phase 1 | Relay can't verify additional keys upstream; trusted sources guarantee correct env→key mapping | Manual multi-key with verification (no suitable verification endpoint; staleness problem; Phase 2 resolves anyway), opt-in unsafe flag (same concerns) | -| Single upstream connection per env on the anchor | Connection-count efficiency at scale; aligns with Phase 2's single-mega-stream model | Multi-client (SDK-2415 PoC approach): trades re-anchor complexity for fan-out at customer scale | -| Anchor by `sdkKey.value` byte-match (no `isDefault` flag) | Single source of truth; matches what RAC already emits | `isDefault` flag (would require backend wire change and dual sources of truth) | -| Per-key `expiry` (Unix-ms) on array entries | Confirmed real format from producers; reuses existing ticker | Per-env single deprecated slot (today's model — doesn't scale to multi-key) | -| Trust the array on expiry disagreement | Simpler invariant; legacy field becomes write-only shim | Take whichever is later, hard-fail on disagreement (more complex, no clear value) | -| Events collapse to anchor per kind, no per-key attribution | Keys are secrets — not appropriate as analytics tags; LD provides better tagging mechanisms | Per-key attribution (would multiply event machinery N×) | -| Diagnostic events keep verbatim-proxy behavior | Preserves operational debug value (which SDK reported); minimal code change | Collapse diagnostic to anchor (loses debug signal); metadata-header (long-term direction, out of Phase 1 scope) | -| `ReconcileCredentials` API replaces `UpdateCredential` everywhere | Atomic semantics; single API surface; no external consumers to preserve | Keep both methods (two ways to do the same thing); stateful batching (non-idiomatic Go) | -| On malformed payload: preserve previous state **+ reconnect RAC stream with jitter** | Loud, safe, atomic — and forces backend to push a fresh `put`, since RAC has no NAK | Soft-fall-back to `sdkKeys[0]` (silent, order-dependent); refuse to serve until next valid update (disruptive); preserve-without-reconnect (backend stays out of sync until something else triggers a refresh) | -| Order of operations: add → re-anchor → remove | Accepted set is a superset during transition; downstream survives | Remove first (downstream-availability window); concurrent (race-prone); atomic batch (atomicity breaks at goroutine boundary) | -| Re-anchor: keep old store/anchor authoritative until new client `Initialized()` | The in-memory store is rebuilt empty on new-client construction (PoC H1, H5); must keep old serving until new is ready | Require persistent store (limits feature to a subset); decouple store from client lifecycle (much bigger refactor) | -| Re-anchor: validate new client `Initialized()` before flipping the anchor pointer; rollback on failure | Avoids mid-swap nil `GetClient()` (PoC H6) and stranded-anchor on init failure (PoC H7) | Flip-then-init (today's broken behavior); accept the gap (visible to customers) | -| Manual config stays single-key in Phase 1 | Same trusted-source reasoning as above | Verify-on-startup, opt-in unsafe flag (rejected for the same reasons in §1) | - ---- - -## 14. Resolved questions - -All design-blocking questions have been answered. - -- **Q5** (RAC propagation SLA for `sdkKey.value` changes): **Real-time.** Same delivery semantics as flag eval / delivery in the SDK. -- **Q6** (Behavior on malformed RAC payload): **Preserve previous accepted set + reconnect the RAC stream with jitter** to force a fresh `put` from the backend (the backend believes the patch was applied because RAC has no NAK channel). See §9. -- **Q7** (Legacy `sdkKey.expiring{}` vs per-key `expiry` disagreement policy): **Trust the array.** Legacy field is a write-only back-compat shim; new relays ignore it on read. -- **Q8** (Per-credential downstream tracking for targeted disconnect): **Already implemented** — today's rotation/disconnect path uses it. T1.c builds on the existing tracking; does not have to construct new infrastructure. -- **Q11** (Customer downgrade story): **No mitigation work.** Documented in release notes; customers reverting from Phase 1 understand they lose multi-key support. - -T0 PoC findings (re-anchoring mechanics) are recorded in §7 and in [`phase1-T0-reanchor-poc-findings.md`](./phase1-T0-reanchor-poc-findings.md). - ---- - -## 15. Glossary - -- **Anchor**: the SDK key the singular `sdkKey.value` field designates. Used for the upstream connection and as the event-dispatcher's stored `authKey`. -- **Accepted set**: all SDK keys + mobile keys + env IDs an environment will accept for downstream-SDK authentication. Includes the anchor. -- **Identifier** (wire `key`): the non-secret human-readable name of a credential. Used in API paths and status display. -- **Credential / value** (wire `value`): the actual secret string (e.g. `sdk-xxxx-...`). What relay's existing types `SDKKey`/`MobileKey`/`SDKCredential` refer to. -- **RAC** (Relay Auto Config): the push channel by which LaunchDarkly delivers environment configuration to enterprise relays. SSE over HTTPS. -- **Offline archive**: a file generated by LaunchDarkly tooling carrying the same `EnvironmentRep` shape. Reloaded periodically. -- **Re-anchor**: swap the upstream SDK client when `sdkKey.value` changes. Single trigger for the swap mechanism. -- **Trusted source**: a LaunchDarkly-generated configuration source (RAC or offline archive). Guaranteed to carry only the environment's real keys. diff --git a/.agent-docs/concurrent-keys/phase1-plan.md b/.agent-docs/concurrent-keys/phase1-plan.md deleted file mode 100644 index 1197b754..00000000 --- a/.agent-docs/concurrent-keys/phase1-plan.md +++ /dev/null @@ -1,557 +0,0 @@ -# Phase 1 — Concurrent SDK Keys in Relay Proxy: Plan - -**Epic**: [SDK-2453](https://launchdarkly.atlassian.net/browse/SDK-2453) -**Companion**: [`phase1-design.md`](./phase1-design.md) (architecture, decisions, model) - -This document covers *how* we ship Phase 1: branching, sequencing, tasks, dependencies, estimates, test strategy, and rollout. - ---- - -## 1. Branching strategy - -**Long-lived feature branch off v8: `feat/concurrent-keys`.** Team convention for feature branches across LD repos — recognizable to other team members. - -- All sub-task PRs target `feat/concurrent-keys`, not v8 directly. -- Sub-PR branches follow Aaron's convention: `aaronz//`, where `` is the **specific sub-task ticket** (e.g. `SDK-2521` for T1.0), **not the epic SDK-2453**. Example: `aaronz/SDK-2521/T1.0-rotate-with-grace-mobile-fix`. -- **Regularly merge `v8` into `feat/concurrent-keys`** (weekly) to surface conflicts incrementally rather than at the end. -- **Final merge** `feat/concurrent-keys` → v8 happens when the feature is fully ready and verified, as a single feat commit. - -### Worktree - -```bash -git worktree add ../ld-relay-wt-feat-concurrent-keys -b feat/concurrent-keys v8 -``` - -This document is committed in that worktree at `.agent-docs/concurrent-keys/phase1-plan.md`. - ---- - -## 2. Wave breakdown - -Three waves. Wave 3 is release-time work: clean up project scaffolding, merge `feat/concurrent-keys` to v8 and release, then merge-forward to v9 when v9 is ready (possibly weeks or months later). - -| Wave | Theme | When | -|---|---|---| -| **Wave 1** | Foundations: PoC, data structures, wire types, test infrastructure | Immediately, multiple sub-tasks in parallel | -| **Wave 2** | Core implementation: API surface change, re-anchor mechanism, peripheral re-wiring, end-to-end integration | After PoC findings + Wave 1 data structures land | -| **Wave 3** | Release: code cleanup → merge to v8 → publish release → merge-forward to v9 | Cleanup + v8 release happen as soon as Wave 2 completes; merge-forward to v9 is calendar-deferred | - -The "single-key behavior unchanged at every PR boundary" invariant is the load-bearing testable property. Every sub-PR must preserve it. - ---- - -## 2.1 Milestone view (delivery-oriented) - -§2's waves are about *sequencing*; this is about *demonstrable capability*. The work groups into five milestones; each is "done" when its end-to-end acceptance scenarios (§7) pass. - -| Milestone | Capability | Tickets | Status | -|---|---|---|---| -| **M1 — Parse the new payload** | Read `sdkKeys[]`/`mobileKeys[]` (+ per-key expiry); old relays ignore the new fields | SDK-2545 (T3.a) | ✅ Done (#702) | -| **M2 — Anchor + multi-key auth** | Accept N keys; one upstream connection via the anchor; every key authenticates downstream | SDK-2538 (foundation) + SDK-2546 (helper, #713) + SDK-2547 (wire handlers) | In progress | -| **M3 — Expiry & rotation** | Grace-period expiry/deprecation; re-anchoring; targeted downstream disconnects | SDK-2539 (ticker, #714) + SDK-2542 (re-anchor) + SDK-2577 (rotation leak cleanup) | Not started | -| **M4 — Big segments after re-anchor** | Big-segment sync survives a re-anchor | SDK-2543 (T2.d) | Not started | -| **M5 — Handler fan-out** | One handler per `(filter, provider)` instead of per credential | SDK-2544 (T2.e) | Not started | - -### Decision (2026-06-23): finish M2 before heavy M3; split T1.b into foundation + wiring - -To keep PRs reviewable and land a working multi-key milestone before taking on rotation, **T1.b (SDK-2538 / #712) is reduced to a behavior-neutral foundation**: - -- The `AcceptedSet` data model (incl. per-key expiry *fields* — the full payload is represented), the `Rotator.Reconcile` add / set-anchor / remove core, the anchor-only upstream client (absorbed from T2.a / SDK-2540), and the `ReconcileCredentials` API. -- It does **not** remove `UpdateCredential` and does **not** change the action handlers — so it carries zero production behavior change. Existing rotation keeps flowing through `UpdateCredential`. - -**M2 is completed by T3.b (SDK-2546 / #713) + T3.c (SDK-2547):** the helper builds the `AcceptedSet` from the full parsed key arrays, and T3.c wires both handlers to `ReconcileCredentials`, removes `UpdateCredential`, and validates the payload. There is no window where both credential paths are live — the handler switch and the `UpdateCredential` deletion land together in T3.c. - -**M3 owns the rotation behavior:** the grace-period deprecation, the cleanup ticker (SDK-2539), the robust re-anchor mechanism, and the "re-queue an already-accepted key when it becomes the primary/anchor" fixes (both SDK and mobile, SDK-2542). Re-anchoring (changing the anchor) is M3, not M2. - ---- - -## 3. Task list - -Each task has: ticket name, files touched, dependencies, estimates. Acceptance criteria live in the JIRA ticket; rationale lives in [`phase1-design.md`](./phase1-design.md). - -### Wave 1 - -| Task | Files | Depends on | Human | AI agent | -|---|---|---|---|---| -| **T0** — Re-anchoring PoC | `internal/relayenv/` (new test files) | — | 3-5 days | 1-2 days (with iteration) | -| **T1.0** — Remove `RotateWithGrace` mobile-key panic | `internal/credential/rotator.go` | — | 0.5 day | 30 min - 1 hr | -| **T1.a** — Add Rotator accepted-set data structures | `internal/credential/rotator.go`, `credential.go` | T1.0 | 1-2 days | 1-2 hrs | -| **T3.a** — Extend `EnvironmentRep` + verify | `internal/envfactory/*`, possibly `archive_reader.go` | — | 1-2 days | 1-2 hrs | -| **T5.a** — Integration test harness | New test infrastructure dir | — | 2-3 days | 2-3 hrs | -| **T5.b** — Events payload regression test + baseline | `internal/events/*_test.go` (new) | — | 1-2 days | 1-2 hrs | - -**Wave 1 total**: 8.5-14.5 human days; 6-10 AI agent hours. - -### Wave 2 - -| Task | Files | Depends on | Human | AI agent | -|---|---|---|---|---| -| **T1.b** — `ReconcileCredentials` API + migrate call sites + remove `UpdateCredential` | `internal/relayenv/env_context*.go`, both action handlers, tests | T0, T1.a | 2-3 days | 2-4 hrs | -| **T1.c** — Generalize cleanup ticker for per-key expiry + mobile-key disconnects | `internal/credential/rotator.go`, `env_context_impl.go` | T1.b, Q8 verified | 2-3 days | 2-3 hrs | -| **T2.a** — `addCredential` anchor-only client | `internal/relayenv/env_context_impl.go` | T0, T1.b | 1-2 days | 1-2 hrs | -| **T2.b** — `GetClient()` returns anchor | `internal/relayenv/env_context_impl.go` | T2.a | 0.5 day | 30 min - 1 hr | -| **T2.c** — Re-anchor mechanism per PoC findings | `internal/relayenv/env_context_impl.go` | T0, T2.a | 3-5 days | 3-5 hrs | -| **T2.d** — Big-segment sync + `httpconfig` from anchor | `internal/relayenv/env_context_impl.go`, big-segment code | T0 | 2-3 days | 2-3 hrs | -| **T2.e** — Handler fan-out optimization | `internal/relayenv/env_context_impl.go`, stream provider interface | T2.c | 2-3 days | 2-3 hrs | -| **T3.b** — Shared reconcile helper | New helper in `internal/envfactory/` | T1.b | 2-3 days | 2-3 hrs | -| **T3.c** — Wire RAC + offline handlers | `relay/autoconfig_actions.go`, `relay/filedata_actions.go` | T3.a, T3.b, T1.b | 2-3 days | 2-3 hrs | -| **T4** — Status endpoint arrays | `internal/api/status_reps.go`, `relay/endpoints_status.go` | T1.b, T3.c | 1-2 days | 1-2 hrs | - -**Wave 2 total**: 18-30 human days; 18-32 AI agent hours. - -### Wave 3 - -| Task | Files | Depends on | Human | AI agent | -|---|---|---|---|---| -| **T5.f** — Code cleanup: remove project scaffolding before v8 merge | `.agent-docs/concurrent-keys/` (entire directory), `internal/relayenv/env_context_reanchor_test.go`, anything else project-specific | All Wave 2 terminal sub-tasks (T1.c, T2.b, T2.d, T2.e, T4) | 0.5-1 day | 30 min - 1 hr | -| **T5.g** — Merge `feat/concurrent-keys` to v8 + publish release | None (release activity) | T5.f | 0.5-1 day | n/a (release task, not coding) | -| **T5.e** — Merge-forward to v9 | `internal/relayenv/*`, streaming path | T5.g (and calendar — may be weeks/months after the v8 release) | 3-7 days | 1-2 days (with iteration) | - -**Total project**: ~6-11 weeks of full-time human work for code work; the merge-forward to v9 (T5.e) lives on its own calendar that depends on v9 readiness. - ---- - -## 4. Dependency graph - -``` - ┌─→ T1.b ─┬─→ T1.c - │ ├─→ T2.a ─→ T2.b - │ │ └──→ T2.c ─→ T2.e - │ ├─→ T3.b ─→ T3.c - │ └─→ T4 -T0 (PoC) ──────────────────────────────────────── ┤ - ├─→ T2.a (PoC needed for swap mechanism) - ├─→ T2.c - └─→ T2.d - -T1.0 ─→ T1.a ─────────────────────────────────────┘ - -T3.a ─────────────────────────────────────────────┐ - ├─→ T3.c -T3.b ─────────────────────────────────────────────┘ - -T5.a (test harness) — supports all other tasks' tests -T5.b (events regression) — runs continuously after landing - -[Wave 2 terminal nodes: T1.c, T2.b, T2.d, T2.e, T4] ─→ T5.f (cleanup) ─→ T5.g (merge to v8 + release) ─→ T5.e (merge-forward to v9) -``` - -**Critical path** (longest dependency chain): -T1.0 → T1.a → T1.b → T2.a → T2.c → T2.e → T5.f → T5.g → T5.e - -The Wave 2 portion is roughly: 0.5 + 1.5 + 2.5 + 1.5 + 4 + 2.5 = ~12.5 human days at the midpoint. Wave 3 adds ~1 day (cleanup) + ~1 day (merge/release) + 3-7 days (v9 merge-forward, calendar-deferred). Other Wave 2 tasks parallelize off this critical path. - ---- - -## 5. Implementation notes per task - -For each task, JIRA tickets carry the full acceptance criteria. Below are *notes that don't fit cleanly into a ticket* — design rationale, code references, things to watch for. - -### T0 — Re-anchoring PoC - -The PoC validates the swap mechanism that T2.c will implement. It is *the* prerequisite — without it, T2 is speculation. The PoC's deliverable is durable test code that survives into T2. - -Seven hypotheses to validate (each becomes a test): -1. Two clients sharing a `storeAdapter` don't corrupt store invariants. -2. Downstream SSE connections tolerate the swap. -3. Big-segment sync keeps working after re-anchor (or identify what re-wiring is needed). -4. `httpconfig` stays functional after re-anchor. -5. Settle order of operations (start-new → swap pointer → close-old vs. alternatives). -6. Behavior during the swap window. -7. Failure modes: new client init fails. - -### T1.0 — Remove `RotateWithGrace` mobile-key panic - -Today: `rotator.go:168-169` panics with `"programmer error: mobile keys do not support deprecation"`. The panic is a guard against an unsupported API state (the data model has no slot for an expiring mobile key). Removing the panic is small; the slot is added by T1.a. - -### T1.a — Rotator data structures - -Internal fields only. No API change. Existing public methods (`SDKKey()`, `GetCredentials()`, etc.) continue to return what they return today by reading from the new internal state where the single primary maps to a one-element set. - -Reviewer-friendly comment to add at the top of the new fields: `// Consumed by T1.b (ReconcileCredentials API). See .agent-docs/concurrent-keys/phase1-design.md §6.2.` - -### T1.b — `ReconcileCredentials` API - -> **Scope reduced (2026-06-23) — see §2.1.** T1.b is now a behavior-neutral *foundation*: the `AcceptedSet` model (incl. per-key expiry fields), `Rotator.Reconcile` (add / set-anchor / remove), the anchor-only client, and the `ReconcileCredentials` API. It **keeps** `UpdateCredential` and does **not** touch the action handlers. The migration + removal described below moved to **T3.c (SDK-2547)**; the rotation refinements moved to T1.c (SDK-2539) / T2.c (SDK-2542). The original note is retained for context. - -The new method replaces `UpdateCredential` *everywhere* — both call sites migrate in this same PR, and `UpdateCredential` + supporting types are removed. There are no external consumers to preserve. - -Today's API surface (to be removed): - -```go -// internal/relayenv/env_context.go:80-85 -UpdateCredential(update *CredentialUpdate) - -// internal/relayenv/env_context.go:27-36 -type CredentialUpdate struct { - primary credential.SDKCredential - deprecated config.SDKKey - expiry time.Time - now time.Time -} -``` - -The new API (bikeshed the exact signature at PR time; this is illustrative): - -```go -ReconcileCredentials(newSet AcceptedSet, anchor credential.SDKCredential) error -``` - -`AcceptedSet` carries the full new state (server keys + mobile keys with optional per-key expiry). The implementation owns the order of operations (`add → re-anchor → remove`) internally; callers don't sequence. - -**On malformed payload**: `ReconcileCredentials` should signal the malformed condition (return a structured error) so that the caller can both (a) preserve the previous accepted set and (b) trigger a reconnect of the RAC stream with jitter (per design §9). T1.b owns the API contract; T3.b/c own driving the reconnect. - -### T1.c — Cleanup ticker - -Generalize `cleanupExpiredCredentials` (called from `StepTime`) to walk the entire accepted set per kind and drop entries whose `expiry` has passed. The downstream-disconnect logic must handle mobile-key disconnects, not just SDK-key ones. - -**Q8 confirmed by team**: per-credential downstream tracking is *already implemented* — today's rotation/disconnect path uses it. T1.c builds on the existing tracking. No new infrastructure to construct; scope is *narrower* than originally feared. - -### T2.a — `addCredential` anchor-only client - -The switch case at `env_context_impl.go:448-463` currently calls `startSDKClient` for any `config.SDKKey`. Phase 1 narrows this to "only the anchor calls `startSDKClient`." Non-anchor server keys get handlers + `envStreams` + lookup mapping but no upstream client. Mobile keys and env IDs already behave this way. - -### T2.b — `GetClient()` returns anchor - -`GetClient()` at `env_context_impl.go:580-594` already returns `c.clients[c.keyRotator.SDKKey()]`. With anchor-only client construction (T2.a), this becomes "return the only client." Verify behavior in tests; small change. - -### T2.c — Re-anchor mechanism - -> **Added scope (2026-06-23):** T2.c also owns the "re-queue an already-accepted key when it becomes the primary" fixes from SDK-2538 / #712 review. **SDK:** when `Reconcile` moves the anchor onto an already-accepted non-anchor key, re-queue it as an addition so `addCredential` runs the anchor-only setup. **Mobile:** when the primary mobile key switches to an already-accepted mobile key, re-queue it so event forwarding follows, and gate `addCredential`'s mobile side-effect on the primary mobile key (Bugbot "Primary mobile switch skips setup", Medium). The SDK fix currently lives in #712 and moves here with the rotation work. - -The big one. PoC findings (design §7 + [`phase1-T0-reanchor-poc-findings.md`](./phase1-T0-reanchor-poc-findings.md)) turned this from "TBD per PoC" into a concrete specification: - -1. **Build** the new anchor's SDK client (do *not* flip the anchor pointer yet). -2. **Wait** for the new client to report `Initialized() == true`. -3. **Atomically flip** the rotator's anchor pointer. Until this moment, `GetClient()` returns the **old** client and evaluations are served from the old store. -4. **Call `ReplaceCredential`** on the event dispatcher and metrics publisher. -5. **Re-wire** (or recreate) big-segment sync — T2.d owns this piece; T2.c calls into it. -6. **Close** the old upstream client after its grace period elapses for any retained downstream traffic. - -**On new-client init failure**: roll back. Anchor pointer stays on the old key, previous accepted set is preserved, structured error logged, alarm raised. The old client (still alive in its grace period) continues to serve. - -**Why this order matters** (PoC H1, H5, H6, H7): -- Flipping the pointer before the new client is registered leaves `GetClient()` returning nil mid-swap (H6). -- Flipping the pointer on init failure strands the env with no usable client even though the old one is fine (H7). -- The in-memory store is rebuilt empty when the new client constructs (H1, H5) — keeping the old anchor authoritative until `Initialized()` is the only way to avoid an evaluation gap without requiring a persistent store. - -**Awareness for T2.c**: downstream SSE connections survive the swap automatically but will receive *one duplicate `put`* from the new client's initial sync (PoC H2). This is tolerable and expected — don't treat it as a bug. - -### T2.d — Big-segment sync re-wire on re-anchor - -T2.d's single responsibility (after PoC): re-wire big-segment sync when the anchor changes. Choose one approach at PR time: - -- **Recreate**: destroy and reconstruct the `BigSegmentSynchronizer` on each re-anchor. Simpler; loses any in-flight sync state. -- **Replace-credential**: add a method to the synchronizer interface that updates its SDK key in place. Preserves in-flight state; requires a new method on the interface. - -**Recommendation**: recreate, unless we discover in-flight state preservation matters for a specific big-segment customer scenario. Recreate is the easier path; switch to replace-credential only if needed. - -`httpconfig` was previously scoped to this task — **PoC H4 confirmed no `httpconfig` change is needed**. The SDK rebuilds the HTTP config with the new anchor key automatically because relay injects the *builder*, not a pre-built config. Removed from T2.d's scope. - -### T2.e — Handler fan-out optimization - -Refactor the handler-building loop at `env_context_impl.go:268-277`. Today: per `(credential, filter, stream provider)`. After: per `(filter, stream provider)`, with the handler resolving the credential from the request at serving time. - -At Block-scale (50 credentials × 10 filters × 4 stream providers), this is the difference between 2,000 handlers per env and 40 per env. See §6 below for the math. - -### T3.a — `EnvironmentRep` extension - -Add the new array fields and the canonical wire-types comment (see [`phase1-design.md`](./phase1-design.md) §5 for the comment text). Verify `DisallowUnknownFields` is not used in the env-parse path (additive-payload guarantee depends on this). Check whether `archive_reader.go` does its own parsing or consumes `EnvironmentRep` directly (T3.a's scope expands if it parses on its own). - -### T3.b — Shared reconcile helper - -A new helper (in `internal/envfactory/` or similar) that both `autoconfig_actions.go` and `filedata_actions.go` call. Responsibilities: -- Diff the old accepted set against the new one (set-keyed by `value`). -- Detect re-anchor (`sdkKey.value` changed). -- Compute the ordered operation list: `add → re-anchor → remove`. -- **Signal malformed-payload condition** (anchor `value` not in `sdkKeys[]`) as a structured error so the caller can both preserve the previous state *and* trigger an RAC stream reconnect with jitter (design §9). -- Treat the legacy `sdkKey.expiring{}` field as write-only — read only the array. - -### T3.c — Wire both action handlers - -> **This is the PR that completes Milestone 2 (2026-06-23) — see §2.1.** Beyond wiring the handlers, T3.c now also owns (moved from T1.b / SDK-2538): **removing `UpdateCredential` / `CredentialUpdate` and migrating both call sites**, and **building the `AcceptedSet` from the full parsed key arrays** (`params.AcceptedSDKKeys` / `AcceptedMobileKeys`, incl. per-key expiry) rather than the singular fields. Plus the **undefined/malformed-credential validation** below (referenced on #712 as "SDK-2534" — a mis-cite; it belongs here): catch undefined/empty credentials and a structurally-malformed payload (anchor `value` absent from `sdkKeys[]`) at parse/process time and surface a structured error instead of silently dropping. - -Replace `UpdateCredential` calls with the new `ReconcileCredentials` API, via the shared helper. RAC handler and offline handler updates land in one PR (separate commits per Aaron's preference). - -**Malformed-payload handling** (design §9): when the shared helper signals a malformed payload, the RAC handler must (a) preserve the previous accepted set and (b) **disconnect and reconnect the RAC stream with jitter** to force a fresh `put` from the backend. The offline handler preserves state only (no equivalent reconnect since there's no live connection — wait for the next archive reload). - -Test matrix (covered in T3.c's acceptance criteria): -- Add a new key -- Set `expiry` on a non-anchor key -- Set `expiry` on the anchor (re-anchor triggered) -- Remove a key (omit from next patch) -- Rename a key (same `value`, different `key` identifier — no-op for creds) -- De-expiry (remove `expiry` on existing entry — cancel scheduled drop) -- Mixed patch (add + re-anchor + remove) -- Partial-failure reconcile (preserves previous state) -- **Malformed payload triggers RAC reconnect** (RAC handler only); state preserved meanwhile - -### T4 — Status endpoint arrays - -Add `sdkKeys` / `mobileKeys` array fields to the env status response. The arrays carry the **full accepted set, including the anchor / primary mobile key** (mirroring the wire format). Each entry: optional non-secret `Key` identifier + obscured `Value` (via `sdks.ObscureKey`) + optional `Expiry`. Keep scalar `sdkKey` / `mobileKey` — they now designate which array entry is the **anchor** / primary, and stay for back-compat. Keep `expiringSdkKey` (the soonest-expiring non-anchor SDK key) for default-rotation back-compat. - -Array entry order is unspecified — consumers look up by `key`/`value`, not position, and the spec treats every key as equally valid. (Stable ordering was a derived-doc embellishment, not a sourced requirement; tests are order-insensitive.) - -Arrays are always present (never omitted/null). `sdkKeys` always contains at least the anchor; `mobileKeys` may be empty for an environment with no mobile key (e.g. server-side only). - -### T5.a — Integration test harness - -Build a reusable harness: -- **RAC mock**: emits captured payloads, supports `put`/`patch`/`delete` event sequences. -- **Downstream SDK simulator**: simulates an SDK connecting with a credential and consuming a stream. -- **Archive fixture loader**: loads offline-mode archives from disk for the filedata path. - -The harness lands as Wave 1 infrastructure; scenarios accumulate as acceptance tests in the sub-tasks that introduce each feature. - -### T5.b — Events payload regression test - -Capture upstream payloads from v8 under realistic SDK traffic. Assert post-Phase-1 payloads are structurally identical *except* for the credential field. Catches accidental schema drift throughout the project. - -### T5.f — Code cleanup before v8 merge - -Remove all project-specific scaffolding from `feat/concurrent-keys` *before* T5.g merges the branch to v8. The canonical design + plan docs and the PoC test file were useful during development; they shouldn't land on v8. - -What to remove: -- `.agent-docs/concurrent-keys/` — entire directory (this file is one of the things being removed). Save off-branch if you want to keep it for reference. -- `internal/relayenv/env_context_reanchor_test.go` — PoC test file. Verify any useful tests have already been adopted into proper regression test files by T2.c before deleting. -- Any other concurrent-keys-specific scaffolding that may have accumulated. - -What stays: -- Actual feature code. -- Regression tests in properly-named test files (those aren't scaffolding). - -Single PR. Conventional commit: `chore(concurrent-keys): remove project scaffolding before v8 merge`. - -### T5.g — Merge `feat/concurrent-keys` to v8 + publish release - -Final merge. **Squash-merge** as a single `feat:` commit — that commit is what release tooling sees, so it triggers the minor version bump. Suggested squash title: `feat(concurrent-keys): support multiple SDK keys per environment via RAC and offline archive`. - -Steps: -1. Final review of feature branch HEAD; confirm cleanup (T5.f) is in. -2. Squash-merge `feat/concurrent-keys` → v8. -3. Verify release tooling triggers a minor version bump. -4. Publish release notes (three items from §8). - -Not a coding task — this is a release activity. - -### T5.e — Merge-forward to v9 - -Not a `git merge`. Real integration work resolving FDv2 ↔ Phase 1 interactions in `env_context_impl.go` and the streaming path. v9 has FDv2 in it, which touches the same files Phase 1 changes most heavily. Validate against v9's existing test suite plus a subset of Phase 1 tests adapted for v9. - -Timing: calendar-deferred. May happen weeks or months after T5.g (v8 release), depending on when v9 is ready. - ---- - -## 6. Handler fan-out optimization (T2.e) — the math - -The optimization observes: today, relay builds one HTTP handler per `(credential, filter, stream provider)` triple. All handlers in the same `(filter, provider)` slot are byte-identical except for the credential baked in at construction. If we look up the credential from the request at serving time, we share one handler per `(filter, provider)`. - -Today: `handlers per env = C × F × P` where C = credentials, F = filters+1, P = stream providers. -After: `handlers per env = F × P`. - -| Customer profile | C | F | P | Unoptimized | Optimized | -|---|---|---|---|---|---| -| Single-key today (baseline) | 3 | 1 | 4 | 12 | 4 | -| Mid-market Phase 1 multi-key | 8 | 1 | 4 | 32 | 4 | -| **Block-scale (multi-key + multi-filter)** | **50** | **10** | **4** | **2,000 per env** | **40 per env** | - -At ~500 bytes per handler closure, Block-scale unoptimized is ~5 MB; optimized is ~100 KB. Memory itself isn't catastrophic, but secondary costs (setup time on every reconcile, GC pressure, per-request lookup overhead) add up. Block is the named customer driver for this project; shipping unoptimized risks regressing memory characteristics for the customer the project is meant to help. - -We're not gating T2.e on an empirical memory benchmark — the napkin math is sufficient justification. - ---- - -## 7. Test strategy - -### Per-PR - -Every sub-PR runs the full existing test suite via existing CI. The "single-key behavior unchanged" invariant is the testable property. - -**Code-review norm** (replaces the dropped T5.d CI job): feature-branch PRs must run the full test suite. Any test that needs to be removed or modified during Phase 1 must be explicitly justified in the PR description. - -### Distributed tests - -Each sub-task's acceptance criteria include unit and scoped-integration tests for that sub-task. Examples: - -- T1.0: panic-removal unit test. -- T1.a: data-structure tests. -- T1.b: `ReconcileCredentials` unit + integration tests. -- T1.c: cleanup-ticker tests, including per-key expiry and mobile-key disconnect. -- T2.c: re-anchor integration tests (evolved from PoC). -- T3.a: parse format tests + old-relay back-compat test + `DisallowUnknownFields` verification. -- T3.c: the reconcile scenario matrix (add, set-expiry, remove, rename, de-expiry, mixed, partial failure). -- T4: status endpoint scenario tests. - -### Cross-cutting tests - -These live in T5 and run continuously: - -- **T5.a (test harness)**: enables the per-sub-task tests above. -- **T5.b (events payload regression)**: catches schema drift in event payloads. - -### End-to-end acceptance scenarios - -The complete catalog of E2E scenarios that prove Phase 1 works. Each scenario is implemented as an integration test in the sub-task(s) listed in **Owner**. The catalog as a whole is the release-readiness coverage check — if every row's tests pass, the project is functionally complete. - -These scenarios live in code (as integration tests). This list is the *registry* — the single place to ask "what does done look like?" - -#### Multi-key authentication - -| # | Scenario | Owner | -|---|---|---| -| 1 | Env with N SDK keys: every key authenticates downstream SDKs correctly; one upstream connection serves all. | T2.a, T3.c | -| 2 | Env with M mobile keys: every mobile key authenticates downstream SDKs correctly. | T1.c, T3.c | -| 3 | Mixed accepted set (SDK + mobile + env ID): all credentials route to the same env context. | T2.a, T3.c | - -#### Re-anchoring - -| # | Scenario | Owner | -|---|---|---| -| 4 | Voluntary anchor rotation: `sdkKey.value` changes; downstream SSE survives; events continue under the new anchor. | T2.c | -| 5 | Default expiry-driven rotation: backend marks current default as expiring and promotes a new one; relay re-anchors; downstream survives. | T2.c, T1.c | -| 6 | New-anchor client init fails: rollback; anchor pointer stays on old key; previous accepted set preserved; structured error logged + alarmed. | T2.c | -| 7 | Big-segment sync remains functional after re-anchor (recreated or re-wired per T2.d's choice). | T2.d | - -#### Key lifecycle - -| # | Scenario | Owner | -|---|---|---| -| 8 | Add a new key: joins the accepted set; existing downstream SDKs undisturbed. | T3.c | -| 9 | Graceful expiry: non-anchor key with `expiry` set; ticker drops it at the timestamp; *only that key's* downstream SDKs disconnect. | T1.c | -| 10 | Immediate revocation: key omitted from next RAC patch; reconcile drops it now; targeted disconnect. | T1.c, T3.c | -| 11 | De-expiry: existing entry's `expiry` removed in next payload; scheduled drop cancelled. | T3.c | -| 12 | Rename: array entry's `key` identifier changes while `value` is preserved; no credential disturbance; status endpoint reflects the new identifier. | T3.c, T4 | -| 13 | Mixed update (add + re-anchor + remove in a single payload): operations apply in order `add → re-anchor → remove`. | T3.c, T2.c | - -#### Defensive behavior - -| # | Scenario | Owner | -|---|---|---| -| 14 | Malformed RAC payload (`sdkKey.value` not present in `sdkKeys[]`): previous accepted set preserved; structured error + alarm logged; RAC stream disconnects and reconnects with jitter; subsequent fresh `put` from backend restores correct state. | T3.b, T3.c | - -#### Sources - -| # | Scenario | Owner | -|---|---|---| -| 15 | RAC multi-key path: scenarios 1–14 work end-to-end via RAC. | T3.c (RAC handler) | -| 16 | Offline archive multi-key path: scenarios 1–13 work end-to-end via filedata reload. | T3.c (offline handler) | - -#### Backward compatibility - -| # | Scenario | Owner | -|---|---|---| -| 17 | Single-key env behaves identically to v8 (the regression invariant — checked at every PR boundary, not just at release). | All sub-PRs; full test suite | -| 18 | Pre-Phase-1 v8 relay parses new-format payload gracefully (additive guarantee). | T3.a | -| 19 | Events payload schema preserved across Phase 1: every field identical to v8 except the credential. | T5.b | - -#### Observability - -| # | Scenario | Owner | -|---|---|---| -| 20 | Status endpoint: scalar fields = anchor (obscured); arrays = full accepted set incl. anchor/primary; per-key `expiry` visible when present; arrays always present (`sdkKeys` ≥ 1, `mobileKeys` may be empty); order unspecified. | T4 | -| 21 | Analytics events forwarded under the env's anchor key per kind, regardless of which accepted key the request came in on. | T2.c, T5.b | -| 22 | Diagnostic events proxy verbatim under the originating credential (deliberate asymmetry — preserved, not collapsed). | T2.c | - -### Release-readiness checklist - -Before merging `feat/concurrent-keys` to v8 (and again before deploying to production), run through: - -1. All Wave 2 sub-tasks merged and tests passing. -2. **Every scenario in "End-to-end acceptance scenarios" above passes** (the catalog is the explicit coverage check). -3. Events payload regression test (T5.b) passes against the full feature branch. -4. Single-key behavior verified identical to v8's baseline via the full test suite. -5. Status endpoint manually inspected for both single-key and multi-key envs. -6. Defensive payload tests: malformed RAC payload → relay logs + preserves previous state + reconnects RAC stream. - -This is a *checklist*, not a discrete task. Touched at release readiness, not as a separate sub-PR. - ---- - -## 8. Rollout - -### Release notes - -Three customer-facing items to surface: - -1. **"Concurrent SDK keys are available for relays using LaunchDarkly's Relay Auto Config.** Manual configuration continues to support one SDK key, one mobile key, and one environment ID per environment. Multi-key support for manual configuration will arrive in a future major release." -2. **"Events from all SDK keys in an environment appear under the anchor key in LaunchDarkly analytics."** This is consistent with today's single-key behavior but worth calling out because the multi-key model invites the expectation that attribution would split. -3. **"Status endpoint adds `sdkKeys` and `mobileKeys` array fields** showing all accepted keys with non-secret identifiers and obscured credentials. Existing `sdkKey` and `mobileKey` scalars now represent the *anchor* key specifically (the key relay uses for its upstream connection)." - -### External follow-ups - -These are tracked but not part of Phase 1's task list: - -- **Public docs update**: the customer docs at `launchdarkly.com/docs/home/account/environment/keys` currently say *"If you are using the Relay Proxy, it can only use the default SDK key."* Phase 1 invalidates this. Aaron's team doesn't own public docs; Aaron contacts the docs-owning team when Phase 1 is close to shipping. - -### Kill switch - -Phase 1 doesn't introduce a config-level kill switch for concurrent keys. Multi-key behavior is effectively opt-in at the customer level — customers who don't create additional keys in LD's UI see zero behavior change. If a critical issue surfaces, the operational mitigation is: customer rolls back to a pre-Phase-1 build, and SDKs using non-anchor keys lose connectivity until the operator updates either the relay or the LD UI. - -### Customer downgrade story (open question for the team) - -Tracked as Q11. Working assumption: surface in release notes; no relay-side mitigation needed beyond messaging. - ---- - -## 9. Deferred items - -These are intentional non-goals, with notes on what would trigger reconsideration: - -- **Memory benchmark for T2.e** — deferred. Napkin math (§6) is sufficient justification. Revive if teammates push back without empirical data. -- **Verify-on-startup for manual-config keys** — rejected. The full reasoning is in [`phase1-design.md`](./phase1-design.md) §1 / §13. -- **Per-key event attribution** — deliberately not pursued. Would require multiplying event machinery; SDK keys are secrets and not appropriate as analytics tags. Long-term path if ever needed: a non-secret metadata header on diagnostic events. Out of Phase 1 scope. -- **Multi env-ID support** — out of scope for Phase 1, mirroring the backend tech spec which also defers client-side ID migration. -- **Phase 2 mega-stream design** — out of scope for this plan. Phase 2 will get its own design doc. - ---- - -## 10. JIRA structure - -``` -SDK-2453 (Epic) — Relay Proxy Multi Keys Support -├── T0 Re-anchoring PoC [Story] -├── T1 Generalize the credential model [Story] -│ ├── T1.0 Remove RotateWithGrace mobile-key panic [Sub-task] -│ ├── T1.a Add Rotator accepted-set data structures [Sub-task] -│ ├── T1.b ReconcileCredentials API + migrate + remove [Sub-task] -│ └── T1.c Generalize cleanup ticker [Sub-task] -├── T2 Decouple upstream-client lifecycle [Story] -│ ├── T2.a addCredential anchor-only client [Sub-task] -│ ├── T2.b GetClient returns anchor's client [Sub-task] -│ ├── T2.c Re-anchor mechanism per PoC [Sub-task] -│ ├── T2.d Big-segment + httpconfig from anchor [Sub-task] -│ └── T2.e Handler fan-out optimization [Sub-task] -├── T3 Plumb N keys from trusted sources [Story] -│ ├── T3.a Extend EnvironmentRep + verify [Sub-task] -│ ├── T3.b Shared reconcile helper [Sub-task] -│ └── T3.c Wire RAC + offline handlers [Sub-task] -├── T4 Status endpoints [Task] -└── T5 Tests, release, and merge-forward [Story] - ├── T5.a Integration test harness [Sub-task] - ├── T5.b Events payload regression test [Sub-task] - ├── T5.f Code cleanup before v8 merge [Sub-task] - ├── T5.g Merge feat/concurrent-keys to v8 + release [Sub-task] - └── T5.e Merge-forward to v9 [Sub-task] -``` - -**Wave labels**: `wave-1`, `wave-2`, `wave-3` on every sub-task (and `wave-1` on T0 since it has no sub-tasks). - -**Dependencies**: modeled via JIRA `blocks` links. See §4 above for the full graph. - ---- - -## 11. Quick reference - -| Question | Answer | -|---|---| -| Feature branch? | `feat/concurrent-keys` off v8 | -| Sub-PR branches? | `aaronz//` off the feature branch (use the specific sub-task ticket ID, not the epic SDK-2453) | -| Where do canonical docs live? | This file + `phase1-design.md` in `.agent-docs/concurrent-keys/` on the feature branch | -| Where do working notes live? | `docs/agents/phase1-*.md` in the design worktree (gitignored, not on this branch) | -| How is ordering enforced within a `keys change` event? | Add → re-anchor → remove (atomic) | -| What triggers re-anchor? | `sdkKey.value` changed | -| Trusted sources for additional keys? | RAC + offline archive only. Manual config = single-key. | -| Events attribution? | Anchor per kind (collapse). No per-key attribution. | -| Test invariant at every PR boundary? | Single-key behavior identical to v8 | -| Where's the rationale for X decision? | See `phase1-design.md` §13 (Recorded decisions) | -| Open questions still pending? | `phase1-design.md` §14 — Q5, Q6, Q7, Q8, Q11 | diff --git a/internal/autoconfig/stream_manager.go b/internal/autoconfig/stream_manager.go index 2474aa1e..1c77fe87 100644 --- a/internal/autoconfig/stream_manager.go +++ b/internal/autoconfig/stream_manager.go @@ -501,7 +501,7 @@ func (s *StreamManager) dispatchEnvAction(id config.EnvironmentID, rep envfactor // set. It is run at the stream parse boundary — before the rep's version is recorded via Upsert — // mirroring how an unparseable event is handled by gotMalformedEvent. // -// Per design §9 a malformed credential payload must preserve the previous accepted set and force a +// A malformed credential payload must preserve the previous accepted set and force a // stream reconnect (RAC is one-way push with no NAK channel, so the reconnect is what makes the // backend resend a fresh put). Validating here rather than after Upsert is essential: the version is // not advanced, so the fresh put — which carries the same version — is not deduplicated away by the @@ -535,7 +535,7 @@ func (s *StreamManager) applyCachedContent(content *PutContent) { // is on. We will never be processing more than one stream message at the same time. // // handlePut returns true if the stream should be restarted — a malformed credential payload in any of -// the environments triggers a reconnect (design §9), while still processing the well-formed ones. +// the environments triggers a reconnect, while still processing the well-formed ones. func (s *StreamManager) handlePut(content PutContent) bool { // A "put" message represents a full environment set. We will compare them one at a time to the // current set of environments (if any), calling the handler's AddEnvironment for any new ones, diff --git a/internal/autoconfig/stream_manager_errors_test.go b/internal/autoconfig/stream_manager_errors_test.go index dc3a4498..56598cb3 100644 --- a/internal/autoconfig/stream_manager_errors_test.go +++ b/internal/autoconfig/stream_manager_errors_test.go @@ -36,7 +36,7 @@ func eventShouldCauseStreamRestart(t *testing.T, event httphelpers.SSEEvent) { // A credential payload that is valid JSON and a structurally valid event, but whose credential set // cannot be built (e.g. an undefined anchor SDK key, or mobile keys with no designated primary), must // be caught at the parse boundary: the previous state is preserved (no AddEnvironment/UpdateEnvironment -// dispatched) and the stream is restarted so the backend resends a fresh put (design §9). This is +// dispatched) and the stream is restarted so the backend resends a fresh put. This is // verified for both patch and put, since both paths run the validation before the version is recorded. func TestMalformedCredentialPayloadCausesStreamRestart(t *testing.T) { // Each shape is a distinct way BuildAcceptedSet rejects a structurally malformed payload; all ride diff --git a/internal/autoconfigcache/reload_restart_redis_test.go b/internal/autoconfigcache/reload_restart_redis_test.go index 14e509dd..d89185e3 100644 --- a/internal/autoconfigcache/reload_restart_redis_test.go +++ b/internal/autoconfigcache/reload_restart_redis_test.go @@ -7,7 +7,7 @@ package autoconfigcache // process restart: a fresh StreamManager, given the same Redis cache but a config stream that delivers // nothing, reloads the environment from the cache with its sdkKeys[]/mobileKeys[] arrays intact. // -// This is the restart-survival half of the "cache integrity across restart" scenario (SDK-2609 #10); +// This is the restart-survival half of the "cache integrity across restart" scenario; // the malformed-put-preserves-cache half is covered by the StreamManager unit test // TestMalformedCredentialPayloadPreservesEnvironmentCache. It exercises the production classes // (StreamManager + redisStore) against an actual Redis, with no test doubles for the cache itself. diff --git a/internal/relayenv/env_context_handler_fanout_test.go b/internal/relayenv/env_context_handler_fanout_test.go index 6a12ba50..66758bd0 100644 --- a/internal/relayenv/env_context_handler_fanout_test.go +++ b/internal/relayenv/env_context_handler_fanout_test.go @@ -1,6 +1,6 @@ package relayenv -// Tests for T2.e (SDK-2544): stream handlers are no longer built or stored per credential. Instead +// Stream handlers are no longer built or stored per credential. Instead // GetStreamHandler resolves the request's credential to a scoped channel and asks the StreamProvider to // build the handler on demand, scoping it with the env's (immutable) filter key. These tests exercise // that on-demand path directly: that the provider is asked for the right scoped credential, that a valid diff --git a/internal/relayenv/env_context_reanchor_bigsegment_test.go b/internal/relayenv/env_context_reanchor_bigsegment_test.go index 4fa18f20..43437943 100644 --- a/internal/relayenv/env_context_reanchor_bigsegment_test.go +++ b/internal/relayenv/env_context_reanchor_bigsegment_test.go @@ -1,8 +1,8 @@ package relayenv -// Tests for T2.d (SDK-2543): the big-segment synchronizer follows the anchor across a re-anchor. -// TestReanchorPoC_H3_BigSegmentSyncFollowsAnchorOnReAnchor covers the basic "recreated on the new key, -// not yet started" case; these cover the started-continues, rollback, and not-configured cases. +// Tests that the big-segment synchronizer follows the anchor across a re-anchor: recreated on the +// new anchor key, an already-started sync continues while the old one closes, a rolled-back +// re-anchor does not rewire, and a not-configured environment is a no-op. import ( "testing" @@ -150,7 +150,7 @@ func TestReanchorBigSegmentSync_NotConfiguredIsNoOp(t *testing.T) { } // reanchorTestKey3 is a third anchor SDK key, used to drive A->B->C sequential re-anchors. -const reanchorTestKey3 = config.SDKKey("reanchor-poc-new-anchor-3") +const reanchorTestKey3 = config.SDKKey("reanchor-new-anchor-3") // TestReanchorBigSegmentSync_ReanchorBeforeFirstSegmentThenStartsNewSync covers the ordering where a // re-anchor happens BEFORE any big segment has appeared (so the replacement is built but not started), diff --git a/internal/relayenv/env_context_reanchor_helpers_test.go b/internal/relayenv/env_context_reanchor_helpers_test.go new file mode 100644 index 00000000..e3e0f3df --- /dev/null +++ b/internal/relayenv/env_context_reanchor_helpers_test.go @@ -0,0 +1,116 @@ +package relayenv + +// Shared helpers for the re-anchor tests: a driver that re-anchors an environment onto a new SDK +// key, plus recording fakes for the stream-update and big-segment-synchronizer collaborators. These +// are consumed by the re-anchor regression tests in this package (env_context_reanchor_*_test.go) +// and by store_handover_realclient_test.go. + +import ( + "sync" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/bigsegments" + "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/internal/httpconfig" + "github.com/launchdarkly/ld-relay/v8/internal/util" + + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoretypes" + + "github.com/stretchr/testify/require" +) + +// reanchorTestKey2 is the "new anchor" SDK key we re-anchor onto. It is deliberately NOT in the real +// sdk- credential format so it won't trip secret scanners; relay treats SDK keys as opaque +// non-empty strings, so any value works here. +const reanchorTestKey2 = config.SDKKey("reanchor-new-anchor") + +// recordingStreamUpdates is a streams.EnvStreamUpdates that counts the broadcasts it receives, so we +// can observe whether a re-anchor produces duplicate downstream "put"s. +type recordingStreamUpdates struct { + mu sync.Mutex + allDataUpdates int + singleUpdates int + invalidations int +} + +func (r *recordingStreamUpdates) SendAllDataUpdate(_ []ldstoretypes.Collection) { + r.mu.Lock() + r.allDataUpdates++ + r.mu.Unlock() +} + +func (r *recordingStreamUpdates) SendSingleItemUpdate(_ ldstoretypes.DataKind, _ string, _ ldstoretypes.ItemDescriptor) { + r.mu.Lock() + r.singleUpdates++ + r.mu.Unlock() +} + +func (r *recordingStreamUpdates) InvalidateClientSideState() { + r.mu.Lock() + r.invalidations++ + r.mu.Unlock() +} + +func (r *recordingStreamUpdates) allDataCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.allDataUpdates +} + +// reanchor re-anchors env onto newKey while keeping oldKey accepted for a grace hour (the old +// client stays up while the new one is built, then closes when the commit lands). This mirrors the +// backend's default-rotation behavior: the new anchor is non-expiring, the demoted old anchor +// carries an expiry. It drives the time-injectable reconcileCredentials directly so the +// grace-period math is deterministic. +func reanchor(t *testing.T, env EnvContext, newKey, oldKey config.SDKKey, now time.Time) { + t.Helper() + set, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: newKey}). + WithSDKKey(credential.SDKKeyParams{Value: oldKey, Expiry: util.PtrOrNil(now.Add(time.Hour))}). + Build() + require.NoError(t, err) + env.(*envContextImpl).reconcileCredentials(set, now) +} + +// capturingBigSegmentSynchronizerFactory records the SDK key it was constructed with and how many +// times it was invoked, so we can detect whether a re-anchor re-wires big-segment sync. +type capturingBigSegmentSynchronizerFactory struct { + mu sync.Mutex + createCount int + lastSDKKey config.SDKKey + synchronizer *mockBigSegmentSynchronizer +} + +func (f *capturingBigSegmentSynchronizerFactory) create( + _ httpconfig.HTTPConfig, + _ bigsegments.BigSegmentStore, + _ string, + _ string, + _ config.EnvironmentID, + sdkKey config.SDKKey, + _ ldlog.Loggers, + _ string, +) bigsegments.BigSegmentSynchronizer { + f.mu.Lock() + defer f.mu.Unlock() + f.createCount++ + f.lastSDKKey = sdkKey + f.synchronizer = &mockBigSegmentSynchronizer{updateCh: make(chan bigsegments.UpdatesSummary)} + return f.synchronizer +} + +func (f *capturingBigSegmentSynchronizerFactory) snapshot() (int, config.SDKKey) { + f.mu.Lock() + defer f.mu.Unlock() + return f.createCount, f.lastSDKKey +} + +// latest returns the most recently created synchronizer (the current one after a re-anchor rebuild). +func (f *capturingBigSegmentSynchronizerFactory) latest() *mockBigSegmentSynchronizer { + f.mu.Lock() + defer f.mu.Unlock() + return f.synchronizer +} diff --git a/internal/relayenv/env_context_reanchor_test.go b/internal/relayenv/env_context_reanchor_test.go deleted file mode 100644 index e061952f..00000000 --- a/internal/relayenv/env_context_reanchor_test.go +++ /dev/null @@ -1,659 +0,0 @@ -package relayenv - -// Re-anchoring proof-of-concept tests. -// -// These tests validate the upstream SDK-client swap mechanism that the re-anchor implementation builds -// on. Each test answers one of the seven hypotheses about how re-anchoring should behave. They are -// written as durable, executable probes of today's primitives so they survive as regression tests and -// as the executable spec for the re-anchor implementation. -// -// Terminology: "re-anchor" = swapping the single upstream SDK client when sdkKey.value changes. -// Today there is no dedicated re-anchor method; the closest existing path is ReconcileCredentials with -// an expiring (grace-period) key (which rotates the primary SDK key and stands up a new client), so -// several tests drive that path and observe where it falls short of the requirements. - -import ( - "errors" - "net/http" - "sync" - "testing" - "time" - - "github.com/launchdarkly/ld-relay/v8/config" - "github.com/launchdarkly/ld-relay/v8/internal/basictypes" - "github.com/launchdarkly/ld-relay/v8/internal/bigsegments" - "github.com/launchdarkly/ld-relay/v8/internal/credential" - "github.com/launchdarkly/ld-relay/v8/internal/httpconfig" - "github.com/launchdarkly/ld-relay/v8/internal/sdks" - st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" - "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" - "github.com/launchdarkly/ld-relay/v8/internal/store" - "github.com/launchdarkly/ld-relay/v8/internal/streams" - "github.com/launchdarkly/ld-relay/v8/internal/util" - - "github.com/launchdarkly/eventsource" - "github.com/launchdarkly/go-sdk-common/v3/ldlog" - "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" - ld "github.com/launchdarkly/go-server-sdk/v7" - "github.com/launchdarkly/go-server-sdk/v7/ldcomponents" - "github.com/launchdarkly/go-server-sdk/v7/subsystems" - "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" - "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoretypes" - helpers "github.com/launchdarkly/go-test-helpers/v3" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// reanchorTestKey2 is the "new anchor" SDK key we re-anchor onto. It is deliberately NOT in the real -// sdk- credential format so it won't trip secret scanners; relay treats SDK keys as opaque -// non-empty strings, so any value works here. -const reanchorTestKey2 = config.SDKKey("reanchor-poc-new-anchor") - -// recordingStreamUpdates is a streams.EnvStreamUpdates that counts the broadcasts it receives, so we -// can observe whether a re-anchor produces duplicate downstream "put"s. -type recordingStreamUpdates struct { - mu sync.Mutex - allDataUpdates int - singleUpdates int - invalidations int -} - -func (r *recordingStreamUpdates) SendAllDataUpdate(_ []ldstoretypes.Collection) { - r.mu.Lock() - r.allDataUpdates++ - r.mu.Unlock() -} - -func (r *recordingStreamUpdates) SendSingleItemUpdate(_ ldstoretypes.DataKind, _ string, _ ldstoretypes.ItemDescriptor) { - r.mu.Lock() - r.singleUpdates++ - r.mu.Unlock() -} - -func (r *recordingStreamUpdates) InvalidateClientSideState() { - r.mu.Lock() - r.invalidations++ - r.mu.Unlock() -} - -func (r *recordingStreamUpdates) allDataCount() int { - r.mu.Lock() - defer r.mu.Unlock() - return r.allDataUpdates -} - -// sharedStoreFactory is a DataStore configurer that hands back the SAME underlying store instance on -// every Build call. It models a persistent store (Redis/DynamoDB), where the data lives outside the -// process and survives the recreation of the wrapping store on a client swap. -type sharedStoreFactory struct { - store subsystems.DataStore -} - -func (f *sharedStoreFactory) Build(_ subsystems.ClientContext) (subsystems.DataStore, error) { - return f.store, nil -} - -// reanchor re-anchors env onto newKey while keeping oldKey accepted for a grace hour (the old -// client stays up while the new one is built, then closes when the commit lands). This mirrors the -// backend's default-rotation behavior: the new anchor is non-expiring, the demoted old anchor -// carries an expiry. It drives the time-injectable reconcileCredentials directly so the -// grace-period math is deterministic. -func reanchor(t *testing.T, env EnvContext, newKey, oldKey config.SDKKey, now time.Time) { - t.Helper() - set, err := credential.NewAcceptedSetBuilder(). - WithAnchor(credential.SDKKeyParams{Value: newKey}). - WithSDKKey(credential.SDKKeyParams{Value: oldKey, Expiry: util.PtrOrNil(now.Add(time.Hour))}). - Build() - require.NoError(t, err) - env.(*envContextImpl).reconcileCredentials(set, now) -} - -// ----------------------------------------------------------------------------------------------- -// Hypothesis 1: Two SDK clients sharing a storeAdapter don't corrupt store invariants. -// ----------------------------------------------------------------------------------------------- - -func TestReanchorPoC_H1_SharedStoreAdapterRebuildSemantics(t *testing.T) { - featureKind := ldstoreimpl.Features() - flagKey := st.Flag1ServerSide.Flag.Key - - // Original PoC finding: each storeAdapter.Build call constructed a fresh wrapper around a fresh - // underlying store, so the new anchor's client would start empty. The store-handover change - // (SSERelayDataStoreAdapter.Build reusing its existing wrapper, with refcounted Close) inverts - // this: the second client receives the SAME wrapper, with its data still in place. This sub-test - // now asserts the post-fix invariant. - t.Run("in-memory factory reuses the existing store on a second client init (store handover)", func(t *testing.T) { - rec := &recordingStreamUpdates{} - adapter := store.NewSSERelayDataStoreAdapter(ldcomponents.InMemoryDataStore(), rec) - - // First client init populates the store, as the original anchor's client would. - s1, err := adapter.Build(subsystems.BasicClientContext{}) - require.NoError(t, err) - require.NoError(t, s1.Init(st.AllData)) - require.Same(t, s1, adapter.GetStore()) - - got, err := adapter.GetStore().Get(featureKind, flagKey) - require.NoError(t, err) - require.NotNil(t, got.Item, "data should be present after the first client's sync") - - // Second client init = the re-anchor's "start new client" step. - s2, err := adapter.Build(subsystems.BasicClientContext{}) - require.NoError(t, err) - - // Post-fix: Build hands the existing wrapper to the new client and the adapter still points at it. - assert.Same(t, s1, s2, "store handover: Build returns the existing wrapper") - assert.Same(t, s2, adapter.GetStore(), "the adapter still points at the shared wrapper") - - // The wrapper stays initialized and the data survives — no empty-store window for the new anchor. - assert.True(t, adapter.GetStore().IsInitialized(), "the shared store remains initialized") - got2, err := adapter.GetStore().Get(featureKind, flagKey) - require.NoError(t, err) - assert.NotNil(t, got2.Item, "data persists across handover") - }) - - // With a persistent store, the underlying data lives outside the wrapper, so the swap preserves it. - // This is the configuration in which the "shared store" assumption actually holds. - t.Run("shared (persistent) underlying store preserves data across client init", func(t *testing.T) { - underlying, err := ldcomponents.InMemoryDataStore().Build(subsystems.BasicClientContext{}) - require.NoError(t, err) - rec := &recordingStreamUpdates{} - adapter := store.NewSSERelayDataStoreAdapter(&sharedStoreFactory{store: underlying}, rec) - - s1, err := adapter.Build(subsystems.BasicClientContext{}) - require.NoError(t, err) - require.NoError(t, s1.Init(st.AllData)) - - // Re-anchor's "start new client" step. - s2, err := adapter.Build(subsystems.BasicClientContext{}) - require.NoError(t, err) - - // The wrapper is new, but it wraps the SAME underlying store, so data + initialization survive. - assert.True(t, s2.IsInitialized(), "persistent store stays initialized across the swap") - got, err := s2.Get(featureKind, flagKey) - require.NoError(t, err) - assert.NotNil(t, got.Item, "data survives the swap when the underlying store is shared") - }) -} - -// ----------------------------------------------------------------------------------------------- -// Hypothesis 2: Downstream SSE connections tolerate the swap. -// 2a: an open downstream connection survives a re-anchor and keeps receiving events. -// 2b: the new anchor's initial sync re-broadcasts a (duplicate) "put" downstream. -// ----------------------------------------------------------------------------------------------- - -func TestReanchorPoC_H2_DownstreamConnectionSurvivesReAnchor(t *testing.T) { - envConfig := st.EnvClientSide.Config - - fakeBigSegmentStoreFactory := func(config.EnvConfig, config.Config, ldlog.Loggers) (bigsegments.BigSegmentStore, error) { - return bigsegments.NewNullBigSegmentStore(), nil - } - fakeSynchronizerFactory := &mockBigSegmentSynchronizerFactory{} - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - jsClientStreams := streams.NewStreamProvider(basictypes.JSClientPingStream, time.Hour, 0) - clientCh := make(chan *testclient.FakeLDClient, 10) - sdkStartedCh := make(chan EnvContext, 10) - env, err := NewEnvContext(EnvContextImplParams{ - Identifiers: EnvIdentifiers{ConfiguredName: st.EnvClientSide.Name}, - EnvConfig: envConfig, - AllConfig: config.Config{}, - BigSegmentStoreFactory: fakeBigSegmentStoreFactory, - BigSegmentSynchronizerFactory: fakeSynchronizerFactory.create, - ClientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientCh), - SDKBigSegmentsConfigFactory: ldcomponents.BigSegments( - st.ExistingInstance[subsystems.BigSegmentStore](&st.NoOpSDKBigSegmentStore{}), - ), - StreamProviders: []streams.StreamProvider{jsClientStreams}, - ConnectionMapper: mockConnectionMapper{}, - Loggers: mockLog.Loggers, - }, sdkStartedCh) - require.NoError(t, err) - defer env.Close() - - synchronizer := fakeSynchronizerFactory.synchronizer - require.NotNil(t, synchronizer) - - // Wait for the original anchor client and initialize the store so the client-side stream is ready. - <-sdkStartedCh - require.NoError(t, env.GetStore().Init(nil)) - - streamHandler := env.GetStreamHandler(jsClientStreams, envConfig.EnvID) - req, _ := http.NewRequest("GET", "", nil) - st.WithStreamRequest(t, req, streamHandler, func(eventCh <-chan eventsource.Event) { - initEvent := helpers.RequireValue(t, eventCh, time.Minute) - assert.Equal(t, "ping", initEvent.Event()) - if !helpers.AssertNoMoreValues(t, eventCh, 100*time.Millisecond) { - t.FailNow() - } - - // --- Re-anchor while the downstream connection is open. --- - // The connection is keyed on the env ID (a ScopedCredential), independent of the upstream SDK - // key, so swapping the SDK anchor must not disturb it. - start := time.Unix(1000, 0) - reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) - - // The new anchor client comes up on a background goroutine. Credential additions start the client - // with a nil readyCh (so it does NOT signal sdkStartedCh); wait on the credential set instead. - require.Eventually(t, func() bool { - creds := env.GetCredentials() - for _, c := range creds { - if c == reanchorTestKey2 { - return true - } - } - return false - }, time.Second, 10*time.Millisecond, "rotation to the new anchor should be applied") - - // FINDING: the open client-side connection survives the swap and still delivers events. - // T2.d re-anchors the big-segment synchronizer, so a post-re-anchor update arrives on the CURRENT - // (rebuilt) synchronizer, not the retired one (whose channel is now closed). - current := fakeSynchronizerFactory.synchronizer - require.NotSame(t, synchronizer, current, "the synchronizer was rebuilt on re-anchor") - current.updateCh <- bigsegments.UpdatesSummary{SegmentKeysUpdated: []string{"fake-segment-key"}} - pingEvent := helpers.RequireValue(t, eventCh, time.Second) - assert.Equal(t, "ping", pingEvent.Event(), "downstream connection should survive the re-anchor") - }) -} - -func TestReanchorPoC_H2_NewClientInitialSyncRebroadcastsPut(t *testing.T) { - rec := &recordingStreamUpdates{} - adapter := store.NewSSERelayDataStoreAdapter(ldcomponents.InMemoryDataStore(), rec) - - // Original anchor client builds and performs its initial sync -> one downstream "put". - s1, err := adapter.Build(subsystems.BasicClientContext{}) - require.NoError(t, err) - require.NoError(t, s1.Init(st.AllData)) - require.Equal(t, 1, rec.allDataCount()) - - // Re-anchor: the new anchor client builds a fresh store and performs its OWN initial sync. - s2, err := adapter.Build(subsystems.BasicClientContext{}) - require.NoError(t, err) - require.NoError(t, s2.Init(st.AllData)) - - // FINDING: the new anchor's initial sync produces a second full "put" to every connected downstream - // stream. From a downstream SDK's perspective this is a duplicate put. It is tolerable (SDKs apply - // puts idempotently) but the re-anchor implementation must expect it; it is not a corruption. - assert.Equal(t, 2, rec.allDataCount(), "the new anchor's initial sync re-broadcasts a full put") -} - -// ----------------------------------------------------------------------------------------------- -// Hypothesis 3: Big-segment sync after re-anchor. -// ----------------------------------------------------------------------------------------------- - -// capturingBigSegmentSynchronizerFactory records the SDK key it was constructed with and how many -// times it was invoked, so we can detect whether a re-anchor re-wires big-segment sync. -type capturingBigSegmentSynchronizerFactory struct { - mu sync.Mutex - createCount int - lastSDKKey config.SDKKey - synchronizer *mockBigSegmentSynchronizer -} - -func (f *capturingBigSegmentSynchronizerFactory) create( - _ httpconfig.HTTPConfig, - _ bigsegments.BigSegmentStore, - _ string, - _ string, - _ config.EnvironmentID, - sdkKey config.SDKKey, - _ ldlog.Loggers, - _ string, -) bigsegments.BigSegmentSynchronizer { - f.mu.Lock() - defer f.mu.Unlock() - f.createCount++ - f.lastSDKKey = sdkKey - f.synchronizer = &mockBigSegmentSynchronizer{updateCh: make(chan bigsegments.UpdatesSummary)} - return f.synchronizer -} - -func (f *capturingBigSegmentSynchronizerFactory) snapshot() (int, config.SDKKey) { - f.mu.Lock() - defer f.mu.Unlock() - return f.createCount, f.lastSDKKey -} - -// latest returns the most recently created synchronizer (the current one after a re-anchor rebuild). -func (f *capturingBigSegmentSynchronizerFactory) latest() *mockBigSegmentSynchronizer { - f.mu.Lock() - defer f.mu.Unlock() - return f.synchronizer -} - -func TestReanchorPoC_H3_BigSegmentSyncFollowsAnchorOnReAnchor(t *testing.T) { - envConfig := st.EnvMain.Config - - fakeBigSegmentStoreFactory := func(config.EnvConfig, config.Config, ldlog.Loggers) (bigsegments.BigSegmentStore, error) { - return bigsegments.NewNullBigSegmentStore(), nil - } - capturing := &capturingBigSegmentSynchronizerFactory{} - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - clientCh := make(chan *testclient.FakeLDClient, 10) - env, err := NewEnvContext(EnvContextImplParams{ - Identifiers: EnvIdentifiers{ConfiguredName: st.EnvMain.Name}, - EnvConfig: envConfig, - AllConfig: config.Config{}, - BigSegmentStoreFactory: fakeBigSegmentStoreFactory, - BigSegmentSynchronizerFactory: capturing.create, - ClientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientCh), - SDKBigSegmentsConfigFactory: ldcomponents.BigSegments( - st.ExistingInstance[subsystems.BigSegmentStore](&st.NoOpSDKBigSegmentStore{}), - ), - ConnectionMapper: mockConnectionMapper{}, - Loggers: mockLog.Loggers, - }, nil) - require.NoError(t, err) - defer env.Close() - - count, sdkKey := capturing.snapshot() - require.Equal(t, 1, count, "the synchronizer is constructed once at env creation") - require.Equal(t, envConfig.SDKKey, sdkKey, "it is wired to the original anchor's SDK key") - - // Re-anchor onto a new SDK key. - start := time.Unix(1000, 0) - reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) - require.Eventually(t, func() bool { - for _, c := range env.GetCredentials() { - if c == reanchorTestKey2 { - return true - } - } - return false - }, time.Second, 10*time.Millisecond, "rotation to the new anchor should be applied") - - // T2.d: the re-anchor recreates the big-segment synchronizer on the NEW anchor key, so its - // poll/stream requests authenticate with the current anchor instead of the retired one. The - // synchronizer bakes its SDK key in at construction and is not restartable, so re-anchoring rebuilds - // it. (reconcileCredentials -> commitReanchor -> reanchorBigSegmentSync runs synchronously.) - count, sdkKey = capturing.snapshot() - assert.Equal(t, 2, count, "the synchronizer is recreated on re-anchor") - assert.Equal(t, reanchorTestKey2, sdkKey, "the new synchronizer references the new anchor key") -} - -// ----------------------------------------------------------------------------------------------- -// Hypothesis 4: httpconfig stays functional after re-anchor. -// ----------------------------------------------------------------------------------------------- - -func TestReanchorPoC_H4_HTTPConfigIsKeyIndependentExceptAuthHeader(t *testing.T) { - loggers := ldlog.NewDisabledLoggers() - key1 := config.SDKKey("sdk-key-one") - key2 := config.SDKKey("sdk-key-two") - - var proxy config.ProxyConfig - var httpC config.HTTPConfig - - c1, err := httpconfig.NewHTTPConfig(proxy, httpC, key1, "user-agent", loggers) - require.NoError(t, err) - c2, err := httpconfig.NewHTTPConfig(proxy, httpC, key2, "user-agent", loggers) - require.NoError(t, err) - - // The ONLY key-dependent artifact is the Authorization default header on the pre-built SDK HTTP - // config. - assert.Equal(t, string(key1), c1.SDKHTTPConfig.DefaultHeaders.Get("Authorization")) - assert.Equal(t, string(key2), c2.SDKHTTPConfig.DefaultHeaders.Get("Authorization")) - - // Everything else (proxy settings, user agent, and the rest of the default headers) is identical - // and key-independent. - h1 := c1.SDKHTTPConfig.DefaultHeaders.Clone() - h2 := c2.SDKHTTPConfig.DefaultHeaders.Clone() - h1.Del("Authorization") - h2.Del("Authorization") - assert.Equal(t, h1, h2, "non-auth default headers are key-independent") - assert.Equal(t, c1.ProxyConfig, c2.ProxyConfig, "proxy config is key-independent") - - // FINDING: httpconfig needs NO re-wire on re-anchor. Relay injects the *builder* - // (SDKHTTPConfigFactory) into ld.Config.HTTP, and the SDK rebuilds the HTTP config with the new - // anchor key when it constructs the new client, so the Authorization header is set correctly for the - // new anchor automatically. The pre-built SDKHTTPConfig / Client() (used for event + big-segment - // transport) is key-independent except for that Authorization header, which those components set per - // request from their own credential rather than reading it from httpconfig. -} - -// ----------------------------------------------------------------------------------------------- -// Hypothesis 5: Order of operations / the in-memory store window. -// ----------------------------------------------------------------------------------------------- - -// TestReanchorPoC_H5_StoreSurvivesReAnchor was originally a proof-of-concept test asserting the -// *broken* pre-handover behavior: that re-anchor caused the env's in-memory store to be replaced with -// a fresh, empty one. Once SSERelayDataStoreAdapter.Build was changed to hand over the existing wrapper -// to the new client (refcounted Close), that breakage is gone. This now asserts the post-fix -// invariant — the data store instance survives the re-anchor and keeps its data — and is preserved -// as the executable proof that handover holds end-to-end through env_context. -func TestReanchorPoC_H5_StoreSurvivesReAnchor(t *testing.T) { - featureKind := ldstoreimpl.Features() - flagKey := st.Flag1ServerSide.Flag.Key - envConfig := st.EnvMain.Config - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - clientCh := make(chan *testclient.FakeLDClient, 10) - readyCh := make(chan EnvContext, 1) - env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactoryWithChannel(true, clientCh), mockLog.Loggers, readyCh) - defer env.Close() - - require.Equal(t, env, requireEnvReady(t, readyCh)) - client1 := requireClientReady(t, clientCh) - require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) - - // Populate the store as the original anchor's client would have via its stream sync. - require.NoError(t, env.GetStore().Init(st.AllData)) - oldStore := env.GetStore() - got, err := oldStore.Get(featureKind, flagKey) - require.NoError(t, err) - require.NotNil(t, got.Item) - - // Re-anchor onto a new key (old key kept accepted for a grace hour; the old client serves while - // the new one is built and closes once the commit lands -- the "start-new-before-close-old" ordering). - start := time.Unix(1000, 0) - reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) - - client2 := requireClientReady(t, clientCh) - require.Eventually(t, func() bool { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond, - "GetClient should return the new anchor's client once it is registered") - - // Post-fix: the adapter handed the existing wrapper to the new client, so the env's store is the - // same instance, still initialized, and the data is intact. There is no empty-store window for the - // new anchor. - newStore := env.GetStore() - assert.Same(t, oldStore, newStore, "the data store instance survives re-anchor (store handover)") - assert.True(t, newStore.IsInitialized(), "the store stays initialized across re-anchor") - got2, err := newStore.Get(featureKind, flagKey) - require.NoError(t, err) - assert.NotNil(t, got2.Item, "data is preserved across re-anchor") -} - -// TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow validates the reviewer suggestion that, because -// relay owns the data store implementation (it hands the SDK a single storeAdapter), the re-anchor can -// hand the existing store over to the new client instead of letting it build a fresh one. Modeled here -// by a DataStoreFactory that returns the same underlying store on every Build; the production change -// is to make SSERelayDataStoreAdapter reuse its store across the swap. With handover the new anchor's -// client sees the populated, initialized store immediately -- no empty-store window (contrast -// TestReanchorPoC_H5_InMemoryStoreIsWipedByReAnchor). -// -// CAVEAT for the implementation (not reproducible with the fake client, so documented here): -// streamUpdatesStoreWrapper.Close() closes the underlying store. If the new client wraps the -// SAME underlying store, closing the retiring client must NOT close it -- the store's lifecycle has to -// be owned by the adapter, not by the client being retired. -func TestReanchorPoC_H5_StoreHandoverAvoidsEmptyWindow(t *testing.T) { - featureKind := ldstoreimpl.Features() - flagKey := st.Flag1ServerSide.Flag.Key - envConfig := st.EnvMain.Config - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - // A store factory that hands the same underlying store to every client (the "handover" model). - underlying, err := ldcomponents.InMemoryDataStore().Build(subsystems.BasicClientContext{}) - require.NoError(t, err) - handoverFactory := &sharedStoreFactory{store: underlying} - - clientCh := make(chan *testclient.FakeLDClient, 10) - readyCh := make(chan EnvContext, 1) - env, err := NewEnvContext(EnvContextImplParams{ - Identifiers: EnvIdentifiers{ConfiguredName: envName}, - EnvConfig: envConfig, - ClientFactory: testclient.FakeLDClientFactoryWithChannel(true, clientCh), - DataStoreFactory: handoverFactory, - ConnectionMapper: mockConnectionMapper{}, - Loggers: mockLog.Loggers, - }, readyCh) - require.NoError(t, err) - defer env.Close() - - require.Equal(t, env, requireEnvReady(t, readyCh)) - client1 := requireClientReady(t, clientCh) - require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) - - // Populate the store as the original anchor's client would have. - require.NoError(t, env.GetStore().Init(st.AllData)) - - // Re-anchor onto a new key. - start := time.Unix(1000, 0) - reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) - - client2 := requireClientReady(t, clientCh) - require.Eventually(t, func() bool { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond) - - // FINDING: with store handover there is no empty-store window -- the new client's store is still - // initialized and still holds the data, because the underlying store was reused rather than rebuilt. - newStore := env.GetStore() - assert.True(t, newStore.IsInitialized(), "handed-over store stays initialized across the re-anchor") - got, err := newStore.Get(featureKind, flagKey) - require.NoError(t, err) - assert.NotNil(t, got.Item, "data is preserved across the re-anchor when the store is handed over") -} - -// ----------------------------------------------------------------------------------------------- -// Hypothesis 6: Behavior during the swap window (requests arriving mid-swap). -// ----------------------------------------------------------------------------------------------- - -// TestReanchorPoC_H6_AnchorHoldsUntilNewClientReady is the inversion of the original H6 finding. -// The originally-observed broken behavior: reconcileCredentials flipped the rotator's anchor -// synchronously and then built the new client asynchronously, opening a window where GetClient() -// (== clients[AnchorKey()]) returned nil for the not-yet-registered new key. The fix builds the new -// client synchronously and only commits the anchor once it reports Initialized, so no such nil window -// exists: while the new client is still building, the anchor stays on the old key and GetClient() keeps -// returning the old, still-serving client. -func TestReanchorPoC_H6_AnchorHoldsUntilNewClientReady(t *testing.T) { - envConfig := st.EnvMain.Config - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - clientCh := make(chan *testclient.FakeLDClient, 10) - inner := testclient.FakeLDClientFactoryWithChannel(true, clientCh) - - // gate blocks construction of the NEW anchor's client so we can observe the swap window - // deterministically. entered signals that the synchronous build has reached the factory (and is now - // blocked on gate) — at which point the re-anchor is mid-flight but has not yet committed. - gate := make(chan struct{}) - entered := make(chan struct{}, 1) - gatedFactory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { - if sdkKey == reanchorTestKey2 { - entered <- struct{}{} - <-gate - } - return inner(sdkKey, cfg, timeout) - } - - readyCh := make(chan EnvContext, 1) - env := makeBasicEnv(t, envConfig, gatedFactory, mockLog.Loggers, readyCh) - defer env.Close() - - require.Equal(t, env, requireEnvReady(t, readyCh)) - client1 := requireClientReady(t, clientCh) - require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) - - // The re-anchor is synchronous, so the reconcile blocks in the gated factory until we release it. - // Drive it on a background goroutine and build the set up front (keeping require off that goroutine). - start := time.Unix(1000, 0) - set, err := credential.NewAcceptedSetBuilder(). - WithAnchor(credential.SDKKeyParams{Value: reanchorTestKey2}). - WithSDKKey(credential.SDKKeyParams{Value: envConfig.SDKKey, Expiry: util.PtrOrNil(start.Add(time.Hour))}). - Build() - require.NoError(t, err) - done := make(chan struct{}) - go func() { - defer close(done) - env.(*envContextImpl).reconcileCredentials(set, start) - }() - - // The build has reached the factory and is blocked: the re-anchor is mid-flight, pre-commit. - <-entered - envImpl := env.(*envContextImpl) - assert.Same(t, client1, env.GetClient(), "GetClient() keeps returning the old client during the build — never nil") - assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "the anchor stays on the old key until the new client is ready") - - // Release the gate; the new client initializes, the anchor commits, and GetClient() advances. - close(gate) - <-done - client2 := requireClientReady(t, clientCh) - require.Eventually(t, func() bool { return env.GetClient() == client2 }, time.Second, 10*time.Millisecond, - "GetClient() returns the new client once it is registered and the anchor commits") - assert.Equal(t, reanchorTestKey2, envImpl.keyRotator.AnchorKey()) -} - -// ----------------------------------------------------------------------------------------------- -// Hypothesis 7: Failure modes — new client init fails. -// ----------------------------------------------------------------------------------------------- - -// TestReanchorPoC_H7_FailedNewClientRollsBackToOldAnchor is the inversion of the original H7 finding. -// The originally-observed broken behavior: a failed new-client init left the anchor already flipped to -// a key with no client, so GetClient() returned nil and the environment was broken even though the old -// client was still alive. The fix builds and validates the new client before committing: on init -// failure it does NOT commit the anchor, surfaces the error, and leaves the old anchor authoritative -// (its client keeps serving). This is the all-or-nothing atomicity requirement applied to re-anchor. -func TestReanchorPoC_H7_FailedNewClientRollsBackToOldAnchor(t *testing.T) { - envConfig := st.EnvMain.Config - fakeErr := errors.New("new anchor client failed to initialize") - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - clientCh := make(chan *testclient.FakeLDClient, 10) - inner := testclient.FakeLDClientFactoryWithChannel(true, clientCh) - - // Succeed for the original anchor; fail for the new anchor. - failingFactory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { - if sdkKey == reanchorTestKey2 { - return nil, fakeErr - } - return inner(sdkKey, cfg, timeout) - } - - readyCh := make(chan EnvContext, 1) - env := makeBasicEnv(t, envConfig, failingFactory, mockLog.Loggers, readyCh) - defer env.Close() - - require.Equal(t, env, requireEnvReady(t, readyCh)) - client1 := requireClientReady(t, clientCh) - require.Eventually(t, func() bool { return env.GetClient() == client1 }, time.Second, 10*time.Millisecond) - - // Re-anchor onto a key whose client init fails (old key kept valid for a grace hour). The re-anchor - // is synchronous, so the init failure and rollback are complete by the time reanchor returns. - start := time.Unix(1000, 0) - reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, start) - - // Post-fix: the re-anchor rolled back. The env stays healthy on the old anchor, so GetInitError - // stays nil — setting it would 401 a still-serving env at the request middleware. The failure - // surfaces via a structured Error log instead. The anchor pointer stayed on the old key, whose - // client is still alive and serving, so GetClient() never returns nil and no client is installed - // for the failed new anchor. - assert.NoError(t, env.GetInitError(), "a failed re-anchor must not mark the still-serving env as failed") - mockLog.AssertMessageMatch(t, true, ldlog.Error, "Re-anchor to SDK key .* failed") - assert.Same(t, client1, env.GetClient(), "GetClient() still returns the old anchor's client after rollback") - envImpl := env.(*envContextImpl) - assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "the anchor pointer stays on the old key") - envImpl.mu.RLock() - _, oldStillPresent := envImpl.clients[envConfig.SDKKey] - _, newInstalled := envImpl.clients[reanchorTestKey2] - envImpl.mu.RUnlock() - assert.True(t, oldStillPresent, "the old anchor's client is preserved") - assert.False(t, newInstalled, "no client is installed for the failed new anchor") -} diff --git a/internal/relayenv/store_handover_realclient_test.go b/internal/relayenv/store_handover_realclient_test.go index 016b9ec7..3e5e3aa4 100644 --- a/internal/relayenv/store_handover_realclient_test.go +++ b/internal/relayenv/store_handover_realclient_test.go @@ -1,8 +1,8 @@ package relayenv -// Spike verifying the real ld.LDClient's Close() behavior against the SSERelayDataStoreAdapter / -// streamUpdatesStoreWrapper pair, which the fake-client PoC could not exercise. This is the single -// remaining piece the fake-client PoC could not validate: +// Verifies the real ld.LDClient's Close() behavior against the SSERelayDataStoreAdapter / +// streamUpdatesStoreWrapper pair, which the fake-client tests could not exercise. This is the single +// remaining piece the fake-client tests could not validate: // // > streamUpdatesStoreWrapper.Close() closes the underlying store. With handover the retiring // > and new clients share one underlying store, so closing the retiring client must NOT close diff --git a/internal/sharedtest/configsource/archive_fixture.go b/internal/sharedtest/configsource/archive_fixture.go deleted file mode 100644 index 24dfb477..00000000 --- a/internal/sharedtest/configsource/archive_fixture.go +++ /dev/null @@ -1,218 +0,0 @@ -package configsource - -import ( - "archive/tar" - "compress/gzip" - "crypto/md5" //nolint:gosec // MD5 is used only for change-detection, not authentication - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - "sort" - "testing" - - "github.com/launchdarkly/ld-relay/v8/config" - "github.com/launchdarkly/ld-relay/v8/internal/envfactory" - - helpers "github.com/launchdarkly/go-test-helpers/v3" -) - -// ArchiveEnvSpec describes one environment to be written into an offline-mode archive. -// Flags and Segments follow the same JSON shape as the archive data files; values should be -// JSON-serializable objects (e.g. from ldbuilders). Both are optional. -type ArchiveEnvSpec struct { - // Rep is the EnvironmentRep written to the metadata file. - Rep envfactory.EnvironmentRep - // DataID is an opaque string stored alongside the metadata. Relay uses it to detect whether - // the data file changed across reloads. Any non-empty string works. - DataID string - // Flags is a map of flag key → JSON-serializable flag object. May be nil. - Flags map[string]any - // Segments is a map of segment key → JSON-serializable segment object. May be nil. - Segments map[string]any -} - -// ArchiveFixtureBuilder builds offline-mode archive files (.tar.gz) for use as Relay's -// FileDataSource. The archive format matches what internal/filedata.ArchiveManager expects: -// an {envID}.json metadata file, an {envID}-data.json flag/segment data file, and a checksum.md5 -// file. Call AddEnv one or more times, then WriteTempFile or WriteFile. -type ArchiveFixtureBuilder struct { - envs []ArchiveEnvSpec -} - -// NewArchiveFixtureBuilder creates an empty builder. -func NewArchiveFixtureBuilder() *ArchiveFixtureBuilder { - return &ArchiveFixtureBuilder{} -} - -// AddEnv adds an environment to the archive. Returns the builder for chaining. -func (b *ArchiveFixtureBuilder) AddEnv(spec ArchiveEnvSpec) *ArchiveFixtureBuilder { - b.envs = append(b.envs, spec) - return b -} - -// WriteTempFile writes the archive to a temporary .tar.gz file and returns its path. The file is -// removed automatically when the test ends. -func (b *ArchiveFixtureBuilder) WriteTempFile(t testing.TB) string { - t.Helper() - f, err := os.CreateTemp("", "ld-relay-archive-*.tar.gz") - if err != nil { - t.Fatalf("ArchiveFixtureBuilder: create temp file: %v", err) - } - path := f.Name() - _ = f.Close() - t.Cleanup(func() { _ = os.Remove(path) }) - b.WriteFile(t, path) - return path -} - -// WriteFile writes the archive to the given path as a .tar.gz file. -func (b *ArchiveFixtureBuilder) WriteFile(t testing.TB, path string) { - t.Helper() - - // Each environment maps to one pair of files ({envID}.json, {envID}-data.json). A duplicate - // EnvID would overwrite those files but be hashed twice in checksum.md5, producing an archive - // that fails filedata's checksum verification. Reject it rather than write a broken archive. - seen := make(map[config.EnvironmentID]bool, len(b.envs)) - for _, spec := range b.envs { - if seen[spec.Rep.EnvID] { - t.Fatalf("ArchiveFixtureBuilder: duplicate environment ID %q; each env must be added once", spec.Rep.EnvID) - } - seen[spec.Rep.EnvID] = true - } - - // Stage files in a temp directory, compute checksum, then tar.gz the result. - helpers.WithTempDir(func(dir string) { - for _, spec := range b.envs { - b.writeEnvFiles(t, dir, spec) - } - envIDs := make([]config.EnvironmentID, 0, len(b.envs)) - for _, spec := range b.envs { - envIDs = append(envIDs, spec.Rep.EnvID) - } - writeArchiveChecksum(t, dir, envIDs) - writeArchiveTarGz(t, path, dir) - }) -} - -// archiveEnvRep mirrors the unexported filedata.archiveEnvironmentRep JSON structure. -type archiveEnvRep struct { - Env envfactory.EnvironmentRep `json:"env"` - DataID string `json:"dataId"` -} - -func (b *ArchiveFixtureBuilder) writeEnvFiles(t testing.TB, dir string, spec ArchiveEnvSpec) { - t.Helper() - - // {envID}.json - metaBytes, err := json.Marshal(archiveEnvRep{Env: spec.Rep, DataID: spec.DataID}) - if err != nil { - t.Fatalf("ArchiveFixtureBuilder: marshal env metadata: %v", err) - } - writeArchiveFile(t, archiveMetadataPath(dir, spec.Rep.EnvID), metaBytes) - - // {envID}-data.json - sdkData := make(map[string]any, 2) - if len(spec.Flags) > 0 { - sdkData["flags"] = spec.Flags - } - if len(spec.Segments) > 0 { - sdkData["segments"] = spec.Segments - } - dataBytes, err := json.Marshal(sdkData) - if err != nil { - t.Fatalf("ArchiveFixtureBuilder: marshal sdk data: %v", err) - } - writeArchiveFile(t, archiveDataPath(dir, spec.Rep.EnvID), dataBytes) -} - -func writeArchiveChecksum(t testing.TB, dir string, envIDs []config.EnvironmentID) { - t.Helper() - paths := make([]string, 0, len(envIDs)*2) - for _, id := range envIDs { - paths = append(paths, archiveMetadataPath(dir, id), archiveDataPath(dir, id)) - } - sort.Strings(paths) - - h := md5.New() //nolint:gosec - for _, p := range paths { - f, err := os.Open(filepath.Clean(p)) - if err != nil { - t.Fatalf("ArchiveFixtureBuilder: open for checksum %s: %v", p, err) - } - if _, err = io.Copy(h, f); err != nil { - _ = f.Close() - t.Fatalf("ArchiveFixtureBuilder: hash %s: %v", p, err) - } - _ = f.Close() - } - writeArchiveFile(t, filepath.Join(dir, "checksum.md5"), h.Sum(nil)) -} - -func writeArchiveTarGz(t testing.TB, destPath, srcDir string) { - t.Helper() - _ = os.Remove(destPath) - f, err := os.OpenFile(filepath.Clean(destPath), os.O_CREATE|os.O_RDWR, 0600) - if err != nil { - t.Fatalf("ArchiveFixtureBuilder: create archive %s: %v", destPath, err) - } - gz := gzip.NewWriter(f) - tw := tar.NewWriter(gz) - - entries, err := os.ReadDir(srcDir) - if err != nil { - t.Fatalf("ArchiveFixtureBuilder: read staging dir: %v", err) - } - for _, entry := range entries { - if entry.IsDir() { - continue - } - srcPath := filepath.Join(srcDir, entry.Name()) - fi, err := os.Stat(srcPath) - if err != nil { - t.Fatalf("ArchiveFixtureBuilder: stat %s: %v", srcPath, err) - } - hdr, err := tar.FileInfoHeader(fi, "") - if err != nil { - t.Fatalf("ArchiveFixtureBuilder: tar header for %s: %v", entry.Name(), err) - } - hdr.Name = entry.Name() // strip any directory prefix - if err = tw.WriteHeader(hdr); err != nil { - t.Fatalf("ArchiveFixtureBuilder: write tar header: %v", err) - } - src, err := os.Open(filepath.Clean(srcPath)) - if err != nil { - t.Fatalf("ArchiveFixtureBuilder: open %s: %v", srcPath, err) - } - if _, err = io.Copy(tw, src); err != nil { - _ = src.Close() - t.Fatalf("ArchiveFixtureBuilder: copy %s into tar: %v", entry.Name(), err) - } - _ = src.Close() - } - if err := tw.Close(); err != nil { - t.Fatalf("ArchiveFixtureBuilder: close tar: %v", err) - } - if err := gz.Close(); err != nil { - t.Fatalf("ArchiveFixtureBuilder: close gzip: %v", err) - } - if err := f.Close(); err != nil { - t.Fatalf("ArchiveFixtureBuilder: close file: %v", err) - } -} - -func archiveMetadataPath(dir string, id config.EnvironmentID) string { - return filepath.Join(dir, fmt.Sprintf("%s.json", string(id))) -} - -func archiveDataPath(dir string, id config.EnvironmentID) string { - return filepath.Join(dir, fmt.Sprintf("%s-data.json", string(id))) -} - -func writeArchiveFile(t testing.TB, path string, data []byte) { - t.Helper() - if err := os.WriteFile(filepath.Clean(path), data, 0600); err != nil { - t.Fatalf("ArchiveFixtureBuilder: write %s: %v", path, err) - } -} diff --git a/internal/sharedtest/configsource/package_info.go b/internal/sharedtest/configsource/package_info.go index ba1ac328..c46f1df8 100644 --- a/internal/sharedtest/configsource/package_info.go +++ b/internal/sharedtest/configsource/package_info.go @@ -1,5 +1,5 @@ -// Package configsource contains test helpers that mock or build the sources Relay loads its -// environment configuration from: the Relay Auto Config (RAC) SSE stream and offline-mode archives. +// Package configsource contains test helpers that mock the source Relay loads its +// environment configuration from: the Relay Auto Config (RAC) SSE stream. // // These live in sharedtest/configsource rather than sharedtest itself because they reference the // envfactory package, which transitively imports relayenv and streams. Putting them in a subpackage diff --git a/internal/sharedtest/configsource/rac_mock.go b/internal/sharedtest/configsource/rac_mock.go index 3a394f35..0760a0e4 100644 --- a/internal/sharedtest/configsource/rac_mock.go +++ b/internal/sharedtest/configsource/rac_mock.go @@ -42,7 +42,7 @@ func NewRACMock(t testing.TB, initialEvent *httphelpers.SSEEvent) *RACMock { // NewRACMockWithReconnect creates a RACMock that serves firstEvent to the first client that connects // and reconnectEvent to the next client — modeling a stream that a client restarts and reconnects to. -// This supports the design's malformed-payload recovery (§9): a rejected patch forces Relay to restart +// This supports malformed-payload recovery: a rejected patch forces Relay to restart // its config stream, and the backend serves a fresh, corrected put on the reconnection. // // Send delivers to the first connection; use it to push the event that forces the restart (e.g. the diff --git a/internal/store/relay_feature_store.go b/internal/store/relay_feature_store.go index 327bac95..16ccb6ed 100644 --- a/internal/store/relay_feature_store.go +++ b/internal/store/relay_feature_store.go @@ -68,7 +68,7 @@ func NewSSERelayDataStoreAdapter( // Build is called by the SDK when the LDClient is being created. // -// Store handover (concurrent-keys re-anchor): if the adapter already holds a wrapper from +// Store handover (re-anchor): if the adapter already holds a wrapper from // a prior client construction, that wrapper is returned again instead of building a fresh one. This // hands the populated, initialized data store over to the new anchor's client during a re-anchor — // no empty-store window, no re-sync. The wrapper refcounts its holders so the underlying store is @@ -134,7 +134,7 @@ func newStreamUpdatesStoreWrapper( } // acquire records an additional holder of the wrapper, used by SSERelayDataStoreAdapter.Build when it -// hands this wrapper to a new client during a concurrent-keys re-anchor. It returns false if the +// hands this wrapper to a new client during a re-anchor. It returns false if the // wrapper has already been fully closed (refCount reached zero and the underlying store was torn // down); the caller must then build a fresh wrapper rather than resurrect a dead one. func (sw *streamUpdatesStoreWrapper) acquire() bool { diff --git a/internal/store/store_rebuild_after_close_test.go b/internal/store/store_rebuild_after_close_test.go index 4d91f8d1..1512b74b 100644 --- a/internal/store/store_rebuild_after_close_test.go +++ b/internal/store/store_rebuild_after_close_test.go @@ -1,6 +1,6 @@ package store -// Regression test for the store-handover refcount contract (concurrent-keys re-anchor). +// Regression test for the store-handover refcount contract (re-anchor). // // SSERelayDataStoreAdapter.Build reuses whatever wrapper is parked in a.store so a re-anchor can hand // the populated store to the new client. The hazard: once the wrapper's refCount reaches zero and its diff --git a/internal/store/store_refcount_test.go b/internal/store/store_refcount_test.go index f74cf586..a6d18cb9 100644 --- a/internal/store/store_refcount_test.go +++ b/internal/store/store_refcount_test.go @@ -1,6 +1,6 @@ package store -// Refcount contract tests for the store-handover wrapper (concurrent-keys re-anchor). +// Refcount contract tests for the store-handover wrapper (re-anchor). // // These cover the two properties the refcount design hinges on but the original suite left // unexercised (multi-agent review, PR #736): Close idempotency past the final release, and safety of a diff --git a/relay/concurrent_keys_auth_test.go b/relay/concurrent_keys_auth_test.go index e3ae1dd7..717f1e7c 100644 --- a/relay/concurrent_keys_auth_test.go +++ b/relay/concurrent_keys_auth_test.go @@ -413,7 +413,7 @@ func TestConcurrentKeysRAC_KeyWithFutureExpiryStillAuthenticates(t *testing.T) { // // Both tests run on the FakeLDClient harness, so they verify the routing/credential-level behavior // of an anchor swap. The real-upstream store handover (avoiding an empty-store window) and -// rollback-on-init-failure robustness is the re-anchor work owned by T2.c. +// rollback-on-init-failure robustness is covered by the re-anchor tests in internal/relayenv. // When a new anchor arrives via RAC (sdkKey.value changes to a brand-new key), the upstream client // swaps to the new anchor and the old anchor is dropped, while the non-anchor key stays accepted. @@ -445,7 +445,7 @@ func TestConcurrentKeysRAC_RotatingAnchorUpdatesUpstreamClient(t *testing.T) { // under it. This uses a real (dummy) SDK client + RAC mock — rather than the FakeLDClient harness — // because FakeLDClient never serves a put on the SSE stream, so it couldn't confirm the connection // was actually established before rotating (and thus couldn't genuinely exercise "rotate while -// connected"). The connection-survival property holds even before the T2.c store-handover work, +// connected"). The connection-survival property holds independently of the store-handover work, // which addresses the empty-store data window during the swap, not connection drops. func TestConcurrentKeysRAC_NonAnchorConnectionSurvivesAnchorRotation(t *testing.T) { putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) diff --git a/relay/concurrent_keys_defensive_test.go b/relay/concurrent_keys_defensive_test.go index 99d83695..68ad5754 100644 --- a/relay/concurrent_keys_defensive_test.go +++ b/relay/concurrent_keys_defensive_test.go @@ -26,7 +26,7 @@ import ( // A malformed patch (anchor absent from sdkKeys[]) is rejected without being applied: the previously // accepted credentials keep authenticating and no new key leaks in. The rejection forces the config // stream to restart, and on the reconnection the backend serves a corrected put whose new key then -// authenticates — completing the preserve-then-recover loop from the design's malformed-payload policy. +// authenticates — completing the preserve-then-recover loop of the malformed-payload recovery policy. func TestConcurrentKeysRAC_MalformedPayloadRecoversAfterReconnect(t *testing.T) { firstPut := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) diff --git a/relay/concurrent_keys_harness_ref_test.go b/relay/concurrent_keys_harness_ref_test.go deleted file mode 100644 index 448dcdde..00000000 --- a/relay/concurrent_keys_harness_ref_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package relay - -// TestConcurrentKeysHarnessReference is the reference integration test for the concurrent-keys -// test helpers. It demonstrates the reusable helpers added to internal/sharedtest working together -// end-to-end: -// -// - configsource.ArchiveFixtureBuilder (offline-mode archive with flag data) -// - configsource.RACMock (RAC SSE server delivering environment configuration) -// - sharedtest.WithStreamRequest + sharedtest.AwaitEventOfType (consuming Relay's SSE stream) -// -// Feature-specific scenario tests live alongside the code they exercise and reuse these helpers. - -import ( - "net/http" - "testing" - "time" - - c "github.com/launchdarkly/ld-relay/v8/config" - "github.com/launchdarkly/ld-relay/v8/internal/envfactory" - "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" - "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/configsource" - "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" - - "github.com/launchdarkly/eventsource" - "github.com/launchdarkly/go-configtypes" - "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" - "github.com/launchdarkly/go-server-sdk-evaluation/v3/ldbuilders" - helpers "github.com/launchdarkly/go-test-helpers/v3" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const ( - harnessEnvID = c.EnvironmentID("harness-ref-env-id") - harnessSDKKey = c.SDKKey("sdk-harness-ref-key-001") - harnessMobileKey = c.MobileKey("mob-harness-ref-key-001") - harnessProjKey = "ref-proj" - harnessFlagKey = "harness-simple-flag" -) - -var harnessEnvRep = envfactory.EnvironmentRep{ - EnvID: harnessEnvID, - EnvKey: "harness-ref", - EnvName: "Harness Reference", - ProjKey: harnessProjKey, - ProjName: "Reference Project", - MobKey: harnessMobileKey, - SDKKey: envfactory.SDKKeyRep{Value: harnessSDKKey}, - Version: 1, -} - -// TestConcurrentKeysHarnessReference exercises the reusable concurrent-keys test helpers. -func TestConcurrentKeysHarnessReference(t *testing.T) { - t.Run("archive fixture + SDK stream: flag data flows through Relay's SSE stream", func(t *testing.T) { - // 1. Build an offline-mode archive containing a single env with a simple boolean flag. - archivePath := configsource.NewArchiveFixtureBuilder(). - AddEnv(configsource.ArchiveEnvSpec{ - Rep: harnessEnvRep, - DataID: "data-v1", - Flags: map[string]any{ - harnessFlagKey: ldbuilders.NewFlagBuilder(harnessFlagKey).Version(1).On(true).Build(), - }, - }). - WriteTempFile(t) - - // 2. Start Relay in offline mode with the real archive manager so the flag data actually - // flows through the data store and into the SSE stream. - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - clientsCreatedCh := make(chan testclient.CapturedLDClient, 4) - cfg := c.Config{} - cfg.OfflineMode.FileDataSource = archivePath - - relay, err := newRelayInternal(cfg, relayInternalOptions{ - loggers: mockLog.Loggers, - clientFactory: testclient.RealLDClientFactoryWithChannel(true, clientsCreatedCh), - // archiveManagerFactory left nil → uses the real filedata.NewArchiveManager - }) - require.NoError(t, err) - defer relay.Close() - - // In offline mode the archive manager loads synchronously, so the client should already - // be in the channel; draining it confirms the environment is ready. - _ = helpers.RequireValue(t, clientsCreatedCh, 3*time.Second, "timed out waiting for SDK client creation") - - // 3. Connect to Relay's server-side SSE stream and verify the initial put event arrives - // and contains the flag. WithStreamRequest drives Relay's handler in-process and cancels - // the request when the action returns, so there is no server/connection teardown to order. - req := sharedtest.BuildRequestWithAuth(http.MethodGet, "/all", harnessSDKKey, nil) - sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { - event := sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) - require.NotNil(t, event) - assert.Contains(t, event.Data(), harnessFlagKey, - "expected put event data to contain the flag key") - }) - }) - - t.Run("RAC mock + SDK stream: Relay discovers env from RAC and serves the SSE stream", func(t *testing.T) { - // 1. Create a RAC mock pre-loaded with a put event for the test environment. - putEvent := configsource.MakeAutoConfigPutEvent(harnessEnvRep) - racMock := configsource.NewRACMock(t, &putEvent) - - // 2. Start Relay configured to use the RAC mock as its config stream. Use CreateDummyClient - // (rather than FakeLDClientFactory) so the data store is initialized with flag data — - // required for Relay to emit a put event on the SSE stream when a client connects. - cfg := c.Config{AutoConfig: c.AutoConfigConfig{Key: testAutoConfKey}} - cfg.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(racMock.URL) - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - relay, err := newRelayInternal(cfg, relayInternalOptions{ - loggers: mockLog.Loggers, - clientFactory: testclient.CreateDummyClient, - }) - require.NoError(t, err) - defer relay.Close() - - // 3. Wait for the env to become available (confirms Relay processed the RAC put event). - h := relayTestHelper{t: t, relay: relay} - env := h.awaitEnvironment(harnessEnvID) - - // awaitEnvironment only waits until the env is discoverable by credential lookup. The SDK - // client is created in a background goroutine (go c.startSDKClient(...)), so GetClient() can - // still be nil at this point. Relay's stream middleware returns 503 (Service Unavailable) - // while GetClient() == nil, which would cause the SSE request below to fail intermittently. - // Wait for the client to be ready before connecting, mirroring the readiness poll in - // internal/relayenv/env_context_impl_test.go (TestChangeSDKKey). - require.Eventually(t, func() bool { - return env.GetClient() != nil - }, 5*time.Second, time.Millisecond*5, "timed out waiting for the SDK client to be ready") - - // 4. Connect to Relay's SSE stream and verify it serves a put event. - req := sharedtest.BuildRequestWithAuth(http.MethodGet, "/all", harnessSDKKey, nil) - sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { - event := sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) - assert.NotNil(t, event, "expected Relay to serve a put event on the SSE stream") - }) - }) -} From c9b3a50eb12ea03146a6e61b8354e0397e66ee9a Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Fri, 24 Jul 2026 14:45:39 -0700 Subject: [PATCH 52/66] chore(release): set version to 8.20.0-rc.1 --- relay/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/version/version.go b/relay/version/version.go index 8399083c..0248e89a 100644 --- a/relay/version/version.go +++ b/relay/version/version.go @@ -2,4 +2,4 @@ package version // Version is the package version -const Version = "8.19.5" // {{ x-release-please-version }} +const Version = "8.20.0-rc.1" // {{ x-release-please-version }} From 839cc5257ef323b5b1de3ec2d6bc700ddacf6369 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Fri, 31 Jul 2026 15:24:52 -0700 Subject: [PATCH 53/66] fix(concurrent-keys): defensive hardening bundle from final review (#765) --- internal/relayenv/env_context_impl.go | 13 +++++++++++-- internal/sdks/client_factory.go | 6 ++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index ce23392f..f28106a4 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -724,6 +724,15 @@ func (c *envContextImpl) reanchor(change *credential.AnchorChange) bool { // half-built client and logged a structured error. initErr is deliberately left untouched on failure: // it feeds the request middleware, and setting it to the new anchor's ErrInitializationFailed would 401 // an env that is serving fine on the previous anchor. +// +// Factory contract (see sdks.ClientFactoryFunc): a factory whose construction builds the environment's +// data store must return a non-nil client even on init failure, so that the client.Close() below releases +// the store reference the build acquired (the store wrapper is refcounted — see streamUpdatesStoreWrapper). +// When client == nil there is no handle to release, and this method cannot safely release one itself: it +// has no signal for whether the store was built, so releasing unconditionally could double-release the +// still-serving previous anchor's store on an early factory error. The real SDK honors the contract (a +// failed init returns a non-nil client; an error before the store is built releases nothing); test +// factories must uphold it too. func (c *envContextImpl) buildNewAnchorClient(newAnchor, previousAnchor config.SDKKey) sdks.LDClientContext { client, err := c.sdkClientFactory(newAnchor, c.sdkConfig, c.sdkInitTimeout) if err != nil || client == nil || !client.Initialized() { @@ -961,8 +970,8 @@ func (c *envContextImpl) GetFilter() config.FilterKey { } func (c *envContextImpl) GetInitError() error { - c.mu.Lock() - defer c.mu.Unlock() + c.mu.RLock() + defer c.mu.RUnlock() return c.initErr } diff --git a/internal/sdks/client_factory.go b/internal/sdks/client_factory.go index 6595dbce..b2d7d3bb 100644 --- a/internal/sdks/client_factory.go +++ b/internal/sdks/client_factory.go @@ -42,6 +42,12 @@ type DataStoreStatusInfo struct { // ClientFactoryFunc is a function that creates the LaunchDarkly client. This is normally // DefaultClientFactory, but it can be changed in order to make configuration changes or for testing. +// +// Store-release contract: a factory whose client construction builds the environment's data store must +// return a non-nil client even when initialization fails, so that the caller's Close() releases the +// (refcounted) store reference the build acquired. Returning (nil, err) after the store has been built +// leaks that reference — the caller has no handle to release it. Returning (nil, err) before the store +// is built is fine, as nothing was acquired. The default SDK factory honors this; test factories must too. type ClientFactoryFunc func(sdkKey config.SDKKey, config ld.Config, timeout time.Duration) (LDClientContext, error) // LDClientConstructor is the function type of the underlying SDK client constructor. From c0fb1b568c8bc9e403cd6cc2895f39f62c716276 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 5 Aug 2026 13:34:07 -0700 Subject: [PATCH 54/66] chore(concurrent-keys): cherry-pick outstanding v8 commits (#787) Brings `feat/concurrent-keys` up to date with the functional commits that landed on `v8`: * PR #734: Add `X-LaunchDarkly-Instance-Id` to the browser CORS allowlist * PR #749: Bound REPORT eval request body size (SEC-8503) * PR #750: Make the usage metrics event publisher capacity configurable * PR #789: Bump otel to 1.44.0 and klauspost/compress to 1.18.7 to patch disclosed CVEs --- config/config.go | 18 +++++++++++ config/config_validation.go | 29 ++++++++++++++++++ config/config_validation_test.go | 39 ++++++++++++++++++++++++ config/test_data_configs_invalid_test.go | 12 ++++++++ config/test_data_configs_valid_test.go | 14 +++++++++ docs/configuration.md | 5 +++ go.mod | 8 ++--- go.sum | 16 +++++----- internal/browser/cors.go | 1 + internal/events/event_publisher.go | 36 +++++++++++++++++++--- internal/events/event_publisher_test.go | 37 ++++++++++++++++++++++ internal/relayenv/env_context_impl.go | 4 ++- relay/relay_endpoints.go | 34 ++++++++++++++++----- relay/relay_endpoints_benchmark_test.go | 3 +- relay/relay_endpoints_test.go | 34 +++++++++++++++++++-- relay/relay_routes.go | 29 +++++++++--------- 16 files changed, 277 insertions(+), 42 deletions(-) create mode 100644 config/config_validation_test.go diff --git a/config/config.go b/config/config.go index 4570e051..5fe62830 100644 --- a/config/config.go +++ b/config/config.go @@ -38,6 +38,17 @@ const ( // DefaultEventCapacity is the default value for EventsConfig.Capacity if not specified. DefaultEventCapacity = 1000 + // DefaultMetricsCapacity is the default value for EventsConfig.MetricsCapacity if not specified. + // This is the maximum queue capacity for the usage-metrics event publisher, which emits one event + // per concurrent unique connection on each flush. It is set well above DefaultEventCapacity because + // high-concurrency nodes routinely exceed 1000 unique connections. + DefaultMetricsCapacity = 10000 + + // DefaultMetricsInitialCapacity is the number of events the usage-metrics publisher queue + // preallocates space for. The queue grows on demand from this size up to MetricsCapacity, so that + // the higher maximum does not reserve all of its memory up front on nodes that never reach it. + DefaultMetricsInitialCapacity = 1000 + // DefaultHeartbeatInterval is the default value for MainConfig.HeartBeatInterval if not specified. DefaultHeartbeatInterval = time.Minute * 3 @@ -95,6 +106,11 @@ const ( // credentials to be revoked nearly instantaneously. It is not necessarily a recommendation. // It likely doesn't make sense to use an interval this frequent in production use-cases. minimumCredentialCleanupInterval = 100 * time.Millisecond + // minimumMetricsCapacity is the smallest value accepted for EventsConfig.MetricsCapacity. Usage + // metrics are how LaunchDarkly reports on account usage, so we do not allow the maximum queue + // capacity to be shrunk below the historical default of 1000; smaller configured values are + // clamped up to this floor. + minimumMetricsCapacity = 1000 ) // DefaultLoggers is the default logging configuration used by Relay. @@ -153,6 +169,7 @@ type MainConfig struct { GracefulShutdownTimeout ct.OptDuration `conf:"GRACEFUL_SHUTDOWN_TIMEOUT"` HeartbeatInterval ct.OptDuration `conf:"HEARTBEAT_INTERVAL"` MaxClientConnectionTime ct.OptDuration `conf:"MAX_CLIENT_CONNECTION_TIME"` + MaxClientRequestBodySize ct.OptBase2Bytes `conf:"MAX_CLIENT_REQUEST_BODY_SIZE"` PingStreamJitterTime ct.OptDuration `conf:"PING_STREAM_JITTER_TIME"` DisconnectedStatusTime ct.OptDuration `conf:"DISCONNECTED_STATUS_TIME"` TLSEnabled bool `conf:"TLS_ENABLED"` @@ -196,6 +213,7 @@ type EventsConfig struct { SendEvents bool `conf:"USE_EVENTS"` FlushInterval ct.OptDuration `conf:"EVENTS_FLUSH_INTERVAL"` Capacity ct.OptIntGreaterThanZero `conf:"EVENTS_CAPACITY"` + MetricsCapacity ct.OptIntGreaterThanZero `conf:"EVENTS_METRICS_CAPACITY"` InlineUsers bool `conf:"EVENTS_INLINE_USERS"` MaxInboundPayloadSize ct.OptBase2Bytes `conf:"EVENTS_MAX_INBOUND_PAYLOAD_SIZE"` } diff --git a/config/config_validation.go b/config/config_validation.go index 75bb6153..6fd17a56 100644 --- a/config/config_validation.go +++ b/config/config_validation.go @@ -17,6 +17,7 @@ var ( errOfflineModePropertiesWithNoFile = errors.New("must specify offline mode filename if other offline mode properties are set") errOfflineModeWithEnvironments = errors.New("cannot configure specific environments if offline mode is enabled") errMaxInboundPayloadSize = errors.New("max inbound payload size must be greater than zero") + errMaxClientRequestBodySize = errors.New("max client request body size must be greater than zero") errAutoConfWithoutDBDisambig = errors.New(`when using auto-configuration with database storage, database prefix (or,` + ` if using DynamoDB, table name) must be specified and must contain "` + AutoConfigEnvironmentIDPlaceholder + `"`) errRedisURLWithHostAndPort = errors.New("please specify Redis URL or host/port, but not both") @@ -30,6 +31,8 @@ var ( errInvalidCredentialCleanupInterval = fmt.Errorf("expired credential cleanup interval must be >= %s", minimumCredentialCleanupInterval) ) +const warnMetricsCapacityBelowMinimum = "configured usage metrics event capacity of %d is below the minimum of %d; using %[2]d instead" + func errEnvironmentWithNoSDKKey(envName string) error { return fmt.Errorf("SDK key is required for environment %q", envName) } @@ -85,6 +88,8 @@ func ValidateConfig(c *Config, loggers ldlog.Loggers) error { validateOfflineMode(&result, c) validateCredentialCleanupInterval(&result, c) validateMaxInboundPayloadSize(&result, c) + validateMaxClientRequestBodySize(&result, c) + validateMetricsCapacity(c, loggers) return result.GetError() } @@ -234,6 +239,30 @@ func validateMaxInboundPayloadSize(result *ct.ValidationResult, c *Config) { } } +func validateMaxClientRequestBodySize(result *ct.ValidationResult, c *Config) { + if c.Main.MaxClientRequestBodySize.IsDefined() { + size := c.Main.MaxClientRequestBodySize.GetOrElse(0) + if size <= 0 { + result.AddError(nil, errMaxClientRequestBodySize) + } + } +} + +// validateMetricsCapacity enforces the minimum queue capacity for the usage-metrics event publisher. +// Rather than fail startup on a too-small value, it clamps the value up to the minimum and warns, so +// that a misconfiguration never prevents Relay from running while still protecting usage telemetry. +func validateMetricsCapacity(c *Config, loggers ldlog.Loggers) { + if !c.Events.MetricsCapacity.IsDefined() { + return + } + if c.Events.MetricsCapacity.GetOrElse(0) < minimumMetricsCapacity { + loggers.Warnf(warnMetricsCapacityBelowMinimum, c.Events.MetricsCapacity.GetOrElse(0), minimumMetricsCapacity) + // This value is a constant known to be greater than zero, so the constructor cannot fail. + clamped, _ := ct.NewOptIntGreaterThanZero(minimumMetricsCapacity) + c.Events.MetricsCapacity = clamped + } +} + func validateConfigDatabases(result *ct.ValidationResult, c *Config, loggers ldlog.Loggers) { normalizeRedisConfig(result, c) diff --git a/config/config_validation_test.go b/config/config_validation_test.go new file mode 100644 index 00000000..571b176f --- /dev/null +++ b/config/config_validation_test.go @@ -0,0 +1,39 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" +) + +func TestValidateMetricsCapacity(t *testing.T) { + t.Run("unset value is left undefined", func(t *testing.T) { + var c Config + mockLog := ldlogtest.NewMockLog() + require.NoError(t, ValidateConfig(&c, mockLog.Loggers)) + assert.False(t, c.Events.MetricsCapacity.IsDefined()) + assert.Len(t, mockLog.GetOutput(ldlog.Warn), 0) + }) + + t.Run("value at or above the minimum is left unchanged", func(t *testing.T) { + var c Config + c.Events.MetricsCapacity = mustOptIntGreaterThanZero(2000) + mockLog := ldlogtest.NewMockLog() + require.NoError(t, ValidateConfig(&c, mockLog.Loggers)) + assert.Equal(t, 2000, c.Events.MetricsCapacity.GetOrElse(0)) + assert.Len(t, mockLog.GetOutput(ldlog.Warn), 0) + }) + + t.Run("value below the minimum is clamped up and warns", func(t *testing.T) { + var c Config + c.Events.MetricsCapacity = mustOptIntGreaterThanZero(500) + mockLog := ldlogtest.NewMockLog() + require.NoError(t, ValidateConfig(&c, mockLog.Loggers)) + assert.Equal(t, minimumMetricsCapacity, c.Events.MetricsCapacity.GetOrElse(0)) + mockLog.AssertMessageMatch(t, true, ldlog.Warn, "usage metrics event capacity of 500 is below the minimum of 1000") + }) +} diff --git a/config/test_data_configs_invalid_test.go b/config/test_data_configs_invalid_test.go index 9d583bbc..23d4bf6f 100644 --- a/config/test_data_configs_invalid_test.go +++ b/config/test_data_configs_invalid_test.go @@ -43,9 +43,21 @@ func makeInvalidConfigs() []testDataInvalidConfig { makeInvalidConfigDynamoDBNoPrefixOrTableName(), makeInvalidConfigDynamoDBAutoConfNoPrefixOrTableName(), makeInvalidConfigMultipleDatabases(), + makeInvalidConfigMaxClientRequestBodySize("0B"), } } +func makeInvalidConfigMaxClientRequestBodySize(size string) testDataInvalidConfig { + c := testDataInvalidConfig{name: "max client request body size " + size} + c.envVarsError = errMaxClientRequestBodySize.Error() + c.envVars = map[string]string{"MAX_CLIENT_REQUEST_BODY_SIZE": size} + c.fileContent = ` +[Main] +MaxClientRequestBodySize = ` + size + ` +` + return c +} + func makeInvalidConfigMissingSDKKey() testDataInvalidConfig { c := testDataInvalidConfig{name: "environment without SDK key"} c.fileContent = ` diff --git a/config/test_data_configs_valid_test.go b/config/test_data_configs_valid_test.go index c668c7b3..9822295d 100644 --- a/config/test_data_configs_valid_test.go +++ b/config/test_data_configs_valid_test.go @@ -49,6 +49,14 @@ func mustOptIntGreaterThanZero(n int) ct.OptIntGreaterThanZero { return o } +func mustOptBase2Bytes(s string) ct.OptBase2Bytes { + o, err := ct.NewOptBase2BytesFromString(s) + if err != nil { + panic(err) + } + return o +} + func newOptURLAbsoluteMustBeValid(urlString string) ct.OptURLAbsolute { o, err := ct.NewOptURLAbsoluteFromString(urlString) if err != nil { @@ -110,6 +118,7 @@ func makeValidConfigAllBaseProperties() testDataValidConfig { IgnoreConnectionErrors: true, HeartbeatInterval: ct.NewOptDuration(90 * time.Second), MaxClientConnectionTime: ct.NewOptDuration(30 * time.Minute), + MaxClientRequestBodySize: mustOptBase2Bytes("5MiB"), DisconnectedStatusTime: ct.NewOptDuration(3 * time.Minute), TLSEnabled: true, TLSCert: "cert", @@ -126,6 +135,7 @@ func makeValidConfigAllBaseProperties() testDataValidConfig { EventsURI: newOptURLAbsoluteMustBeValid("http://events"), FlushInterval: ct.NewOptDuration(120 * time.Second), Capacity: mustOptIntGreaterThanZero(500), + MetricsCapacity: mustOptIntGreaterThanZero(50000), InlineUsers: true, MaxInboundPayloadSize: ct.OptBase2Bytes{}, } @@ -161,6 +171,7 @@ func makeValidConfigAllBaseProperties() testDataValidConfig { "IGNORE_CONNECTION_ERRORS": "1", "HEARTBEAT_INTERVAL": "90s", "MAX_CLIENT_CONNECTION_TIME": "30m", + "MAX_CLIENT_REQUEST_BODY_SIZE": "5MiB", "DISCONNECTED_STATUS_TIME": "3m", "TLS_ENABLED": "1", "TLS_CERT": "cert", @@ -173,6 +184,7 @@ func makeValidConfigAllBaseProperties() testDataValidConfig { "EVENTS_HOST": "http://events", "EVENTS_FLUSH_INTERVAL": "120s", "EVENTS_CAPACITY": "500", + "EVENTS_METRICS_CAPACITY": "50000", "EVENTS_INLINE_USERS": "1", "LD_ENV_earth": "earth-sdk", "LD_MOBILE_KEY_earth": "earth-mob", @@ -203,6 +215,7 @@ ExitAlways = 1 IgnoreConnectionErrors = 1 HeartbeatInterval = 90s MaxClientConnectionTime = 30m +MaxClientRequestBodySize = "5MiB" PingStreamJitterTime = 5m DisconnectedStatusTime = 3m TLSEnabled = 1 @@ -219,6 +232,7 @@ SendEvents = 1 EventsUri = "http://events" FlushInterval = 120s Capacity = 500 +MetricsCapacity = 50000 InlineUsers = 1 [Environment "earth"] diff --git a/docs/configuration.md b/docs/configuration.md index ab1dc530..1f76b3ee 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -58,6 +58,7 @@ For **Duration** settings, the value should be be an integer followed by `ms`, ` | `gracefulShutdownTimeout` | `GRACEFUL_SHUTDOWN_TIMEOUT` | Duration | `30s` | How long the Relay Proxy should wait for active connections to complete before forcefully shutting down when receiving a termination signal. This allows for graceful shutdown of the server, ensuring that in-flight requests are completed. The value should be a duration string like `30s` or `1m`. | | `heartbeatInterval` | `HEARTBEAT_INTERVAL` | Number | `3m` | Interval for heartbeat messages to prevent read timeouts on streaming connections. Assumed to be in seconds if no unit is specified. | | `maxClientConnectionTime` | `MAX_CLIENT_CONNECTION_TIME` | Duration | none | Maximum amount of time that Relay will allow a streaming connection from an SDK client to remain open. _(3)_ | +| `maxClientRequestBodySize` | `MAX_CLIENT_REQUEST_BODY_SIZE` | Unit | none | Maximum size of a `REPORT` request body that Relay will read when evaluating flags for a client-side, mobile, or server-side SDK. _(10)_ | | `disconnectedStatusTime` | `DISCONNECTED_STATUS_TIME` | Duration | `1m` | How long a stream connection can be interrupted before Relay reports the status as "disconnected." _(4)_ | | `tlsEnabled` | `TLS_ENABLED` | Boolean | `false` | Enable TLS on the Relay Proxy. Read: [Using TLS](./tls.md). | | `tlsCert` | `TLS_CERT` | String | | Required if `tlsEnabled` is true. Path to TLS certificate file. | @@ -126,11 +127,15 @@ To learn more, read [Forwarding events](./events.md). | `eventsUri` | `EVENTS_HOST` | URI | _(7)_ | URI for the LaunchDarkly events service | | `flushInterval` | `EVENTS_FLUSH_INTERVAL` | Duration | `5s` | Controls how long the SDK buffers events before sending them back to our server. If your server generates many events per second, we suggest decreasing the flush interval and/or increasing capacity to meet your needs. | | `capacity` | `EVENTS_CAPACITY` | Number | `1000` | Maximum number of events to accumulate for each flush interval. | +| `metricsCapacity` | `EVENTS_METRICS_CAPACITY` | Number | `10000` | Queue capacity for the usage metrics event publisher, which reports connection usage to LaunchDarkly independently of `capacity`. See note _(9)_. | | `inlineUsers` | `EVENTS_INLINE_USERS` | Boolean | `false` | When enabled, individual events (if full event tracking is enabled for the feature flag) will contain all non-private user attributes. | | `maxInboundPayloadSize` | `EVENTS_MAX_INBOUND_PAYLOAD_SIZE` | Unit | _(8)_ | Maximum size of an event payload the Relay Proxy will accept from an SDK. | _(7)_ See note _(1)_ above. The default value for `eventsUri` is `https://events.launchdarkly.com`. _(8)_ The `maxInboundPayloadSize` setting is used to limit the size of the payload that the Relay Proxy will accept from an SDK. This is an optional safety feature to prevent the Relay Proxy from being overwhelmed by a very large payload. The default value is `0B` which provides no restriction on the payload size. The value should be a number followed by a unit: `B` for bytes, `KiB` for kibibytes, `MiB` for mebibytes, `GiB` for gibibytes, `TiB` for tebibytes, `PiB` for pebibytes, or `EiB` for exbibytes. For example, `100MiB` is 100 mebibytes. +_(9)_ The `metricsCapacity` setting controls the queue for usage metrics events, which report connection usage back to LaunchDarkly and are separate from the analytics events governed by `capacity`. The Relay Proxy emits one usage metrics event per concurrent unique connection on each flush, so this should be set to at least the number of concurrent unique connections you expect a single node to serve. This is the maximum capacity: the default is `10000` and the minimum is `1000` (smaller values are clamped up to `1000` with a warning). The queue is an in-memory buffer held per environment that starts small and grows on demand up to this maximum, so its memory footprint tracks the number of concurrent connections actually served rather than the configured maximum. + +_(10)_ The optional `maxClientRequestBodySize` setting limits how much of a `REPORT` evaluation request body the Relay Proxy will read into memory before decoding the context, protecting the process from memory exhaustion caused by oversized request bodies. It applies to the `evalx` context/user endpoints for client-side, mobile, and server-side SDKs. By default it is unset, meaning there is no limit (preserving existing behavior). When set, requests whose body exceeds the limit receive an HTTP `413 Request Entity Too Large` response. The value uses the same units as `maxInboundPayloadSize` (for example, `5MiB`). ### File section: `[Environment "NAME"]` diff --git a/go.mod b/go.mod index 8cf054e4..98ebba00 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,7 @@ require ( require ( github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b - github.com/klauspost/compress v1.18.5 + github.com/klauspost/compress v1.18.7 github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e golang.org/x/crypto v0.53.0 ) @@ -123,9 +123,9 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect diff --git a/go.sum b/go.sum index 476a5716..d909c213 100644 --- a/go.sum +++ b/go.sum @@ -314,8 +314,8 @@ github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3M github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA= github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -520,16 +520,16 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.6 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= diff --git a/internal/browser/cors.go b/internal/browser/cors.go index 10d0b90e..bc1b4105 100644 --- a/internal/browser/cors.go +++ b/internal/browser/cors.go @@ -29,6 +29,7 @@ var DefaultAllowedHeaders = strings.Join([]string{ //nolint:gochecknoglobals "X-LaunchDarkly-User-Agent", "X-LaunchDarkly-Payload-ID", "X-LaunchDarkly-Wrapper", + "X-LaunchDarkly-Instance-Id", events.EventSchemaHeader, events.TagsHeader, }, ",") diff --git a/internal/events/event_publisher.go b/internal/events/event_publisher.go index fd373ced..19c64aab 100644 --- a/internal/events/event_publisher.go +++ b/internal/events/event_publisher.go @@ -103,10 +103,11 @@ type HTTPEventPublisher struct { disableQueue chan interface{} disabled bool - queues map[EventPayloadMetadata]*publisherQueue - capacity int - overflowed bool - lock sync.RWMutex + queues map[EventPayloadMetadata]*publisherQueue + capacity int + initialCapacity int + overflowed bool + lock sync.RWMutex } type eventBatch struct { @@ -159,6 +160,19 @@ func (o OptionCapacity) apply(p *HTTPEventPublisher) error { return nil } +// OptionInitialCapacity specifies how many events to preallocate space for in each event queue. +// The queue still grows on demand (via append) up to OptionCapacity, and events are only dropped +// once OptionCapacity is reached; this option only controls the initial allocation so that a +// publisher with a large capacity does not reserve all of that memory up front. If unset, or not +// smaller than the capacity, the full capacity is preallocated, preserving the original behavior. +type OptionInitialCapacity int + +//nolint:unparam // the error result is required by the OptionType interface +func (o OptionInitialCapacity) apply(p *HTTPEventPublisher) error { + p.initialCapacity = int(o) + return nil +} + // NewHTTPEventPublisher creates a new HTTPEventPublisher. func NewHTTPEventPublisher(authKey credential.SDKCredential, httpConfig httpconfig.HTTPConfig, loggers ldlog.Loggers, options ...OptionType) (*HTTPEventPublisher, error) { closer := make(chan struct{}) @@ -243,10 +257,22 @@ func NewHTTPEventPublisher(authKey credential.SDKCredential, httpConfig httpconf return p, nil } +// initialQueueCapacity returns the number of events to preallocate space for in a new queue. +// It is the smaller of the configured initial capacity and the maximum capacity; when no initial +// capacity is configured (<= 0), the full maximum capacity is preallocated, which is the original +// behavior. The queue can still grow (via append) up to the maximum capacity regardless. +func initialQueueCapacity(capacity, initialCapacity int) int { + if initialCapacity > 0 && initialCapacity < capacity { + return initialCapacity + } + return capacity +} + func (p *HTTPEventPublisher) append(batch eventBatch) { queue := p.queues[batch.metadata] if queue == nil { - queue = &publisherQueue{events: make([]json.RawMessage, 0, p.capacity)} + // The queue still grows up to p.capacity via append regardless of the initial allocation. + queue = &publisherQueue{events: make([]json.RawMessage, 0, initialQueueCapacity(p.capacity, p.initialCapacity))} p.queues[batch.metadata] = queue } available := p.capacity - len(queue.events) diff --git a/internal/events/event_publisher_test.go b/internal/events/event_publisher_test.go index c44e9ba2..6a67ce76 100644 --- a/internal/events/event_publisher_test.go +++ b/internal/events/event_publisher_test.go @@ -189,6 +189,43 @@ func TestHTTPEventPublisherCapacity(t *testing.T) { }) } +func TestInitialQueueCapacity(t *testing.T) { + // Unset initial capacity preallocates the full capacity -- the original behavior, used by the + // analytics publisher, which never sets OptionInitialCapacity. + assert.Equal(t, 1000, initialQueueCapacity(1000, 0)) + assert.Equal(t, 10000, initialQueueCapacity(10000, 0)) + // A smaller initial capacity is used as-is, letting the queue start small and grow. + assert.Equal(t, 1000, initialQueueCapacity(10000, 1000)) + // The initial allocation is never larger than the maximum capacity. + assert.Equal(t, 1000, initialQueueCapacity(1000, 1000)) + assert.Equal(t, 1000, initialQueueCapacity(1000, 5000)) +} + +func TestHTTPEventPublisherInitialCapacityGrowsToCapacity(t *testing.T) { + // With an initial capacity smaller than the (maximum) capacity, the queue must still grow past + // the initial allocation and only drop events once the maximum capacity is reached. + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + handler, requestsCh := httphelpers.RecordingHandler(httphelpers.HandlerWithStatus(202)) + httphelpers.WithServer(handler, func(server *httptest.Server) { + publisher, _ := NewHTTPEventPublisher(config.SDKKey("my-key"), defaultHTTPConfig(), mockLog.Loggers, + OptionBaseURI(server.URL), OptionCapacity(3), OptionInitialCapacity(1)) + defer publisher.Close() + publisher.Publish(EventPayloadMetadata{}, json.RawMessage(`"a"`)) + publisher.Publish(EventPayloadMetadata{}, json.RawMessage(`"b"`)) + publisher.Publish(EventPayloadMetadata{}, json.RawMessage(`"c"`)) + publisher.Publish(EventPayloadMetadata{}, json.RawMessage(`"d"`)) + publisher.Flush() + r := helpers.RequireValue(t, requestsCh, time.Second) + + uncompressed, err := util.DecompressGzipData(r.Body) + assert.NoError(t, err) + + // The queue grew from the initial capacity of 1 up to the capacity of 3, then dropped "d". + m.In(t).Assert(uncompressed, m.JSONStrEqual(`["a","b","c"]`)) + }) +} + func TestHTTPEventPublisherErrorRetry(t *testing.T) { testRecoverableError := func(t *testing.T, errorHandler http.Handler) { mockLog := ldlogtest.NewMockLog() diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index f28106a4..e6c1cef8 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -311,7 +311,9 @@ func NewEnvContext( pubLoggers := envLoggers pubLoggers.SetPrefix(logPrefix + " (usage metrics)") eventsPublisher, err := events.NewHTTPEventPublisher(envConfig.SDKKey, httpConfig, pubLoggers, - events.OptionBaseURI(eventsURI)) + events.OptionBaseURI(eventsURI), + events.OptionCapacity(allConfig.Events.MetricsCapacity.GetOrElse(config.DefaultMetricsCapacity)), + events.OptionInitialCapacity(config.DefaultMetricsInitialCapacity)) if err != nil { return nil, errInitPublisher(err) } diff --git a/relay/relay_endpoints.go b/relay/relay_endpoints.go index 02fec6d1..8eac81ea 100644 --- a/relay/relay_endpoints.go +++ b/relay/relay_endpoints.go @@ -4,6 +4,7 @@ import ( "crypto/sha1" //nolint:gosec // we're not using SHA1 for encryption, just for generating an insecure hash "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -20,6 +21,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/internal/streams" "github.com/launchdarkly/ld-relay/v8/internal/util" + ct "github.com/launchdarkly/go-configtypes" "github.com/launchdarkly/go-jsonstream/v3/jwriter" "github.com/launchdarkly/go-sdk-common/v3/ldcontext" ldevents "github.com/launchdarkly/go-sdk-events/v3" @@ -33,6 +35,7 @@ import ( func getClientSideContextProperties( clientCtx relayenv.EnvContext, sdkKind basictypes.SDKKind, + maxBodySize ct.OptBase2Bytes, req *http.Request, w http.ResponseWriter, ) (ldcontext.Context, bool) { @@ -45,7 +48,24 @@ func getClientSideContextProperties( _, _ = w.Write([]byte("Content-Type must be application/json.")) return ldContext, false } - body, _ := io.ReadAll(req.Body) + bodyReader := req.Body + if maxBodySize.IsDefined() { + bodyReader = http.MaxBytesReader(w, req.Body, int64(maxBodySize.GetOrElse(0))) + } + body, readErr := io.ReadAll(bodyReader) + if readErr != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(readErr, &maxBytesErr) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusRequestEntityTooLarge) + _, _ = w.Write(util.ErrorJSONMsg("Request body exceeds maximum allowed size.")) + return ldContext, false + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write(util.ErrorJSONMsg(readErr.Error())) + return ldContext, false + } contextDecodeErr = json.Unmarshal(body, &ldContext) } else { base64Context := mux.Vars(req)["context"] // this assumes we have used {context} as a placeholder in the route @@ -88,12 +108,12 @@ func pingStreamHandler(streamProvider streams.StreamProvider) http.Handler { // This handler is used for client-side streaming endpoints that require context properties. Currently it is // implemented the same as the ping stream once we have validated the context. -func pingStreamHandlerWithContext(sdkKind basictypes.SDKKind, streamProvider streams.StreamProvider) http.Handler { +func pingStreamHandlerWithContext(sdkKind basictypes.SDKKind, maxBodySize ct.OptBase2Bytes, streamProvider streams.StreamProvider) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { clientCtx := middleware.GetEnvContextInfo(req.Context()) clientCtx.Env.GetLoggers().Debug("Application requested client-side ping stream") - if _, ok := getClientSideContextProperties(clientCtx.Env, sdkKind, req, w); ok { + if _, ok := getClientSideContextProperties(clientCtx.Env, sdkKind, maxBodySize, req, w); ok { clientCtx.Env.GetStreamHandler(streamProvider, clientCtx.Credential).ServeHTTP(w, req) } }) @@ -184,19 +204,19 @@ func bulkEventHandler(sdkKind basictypes.SDKKind, eventsKind ldevents.EventDataK // /sdk/evalx/{envId}/user (REPORT) // /sdk/evalx/users/{context} (GET - with SDK key auth; this is a Relay-only endpoint) // /sdk/evalx/user (REPORT - with SDK key auth; this is a Relay-only endpoint) -func evaluateAllFeatureFlags(sdkKind basictypes.SDKKind) func(w http.ResponseWriter, req *http.Request) { +func evaluateAllFeatureFlags(sdkKind basictypes.SDKKind, maxBodySize ct.OptBase2Bytes) func(w http.ResponseWriter, req *http.Request) { return func(w http.ResponseWriter, req *http.Request) { - evaluateAllShared(w, req, sdkKind) + evaluateAllShared(w, req, sdkKind, maxBodySize) } } -func evaluateAllShared(w http.ResponseWriter, req *http.Request, sdkKind basictypes.SDKKind) { +func evaluateAllShared(w http.ResponseWriter, req *http.Request, sdkKind basictypes.SDKKind, maxBodySize ct.OptBase2Bytes) { clientCtx := middleware.GetEnvContextInfo(req.Context()) client := clientCtx.Env.GetClient() store := clientCtx.Env.GetStore() loggers := clientCtx.Env.GetLoggers() - ldContext, ok := getClientSideContextProperties(clientCtx.Env, sdkKind, req, w) + ldContext, ok := getClientSideContextProperties(clientCtx.Env, sdkKind, maxBodySize, req, w) if !ok { return } diff --git a/relay/relay_endpoints_benchmark_test.go b/relay/relay_endpoints_benchmark_test.go index d6a9d335..f4ab4b1a 100644 --- a/relay/relay_endpoints_benchmark_test.go +++ b/relay/relay_endpoints_benchmark_test.go @@ -10,6 +10,7 @@ import ( "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testenv" + ct "github.com/launchdarkly/go-configtypes" "github.com/launchdarkly/go-sdk-common/v3/lduser" "github.com/launchdarkly/go-sdk-common/v3/ldvalue" "github.com/launchdarkly/go-server-sdk-evaluation/v3/ldbuilders" @@ -53,6 +54,6 @@ func BenchmarkEvaluateAllFlags(b *testing.B) { for i := 0; i < b.N; i++ { req := buildPreRoutedRequest("REPORT", userData, headers, nil, ctx) resp := httptest.NewRecorder() - evaluateAllFeatureFlags(basictypes.JSClientSDK)(resp, req) + evaluateAllFeatureFlags(basictypes.JSClientSDK, ct.OptBase2Bytes{})(resp, req) } } diff --git a/relay/relay_endpoints_test.go b/relay/relay_endpoints_test.go index ddb3142a..222ed631 100644 --- a/relay/relay_endpoints_test.go +++ b/relay/relay_endpoints_test.go @@ -12,6 +12,7 @@ import ( st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testenv" + ct "github.com/launchdarkly/go-configtypes" "github.com/launchdarkly/go-test-helpers/v3/jsonhelpers" "github.com/gorilla/mux" @@ -35,7 +36,7 @@ func TestReportFlagEvalFailsWithUninitializedClientAndStore(t *testing.T) { ctx := testenv.NewTestEnvContext("", false, st.MakeStoreWithData(false)) req := buildPreRoutedRequest("REPORT", []byte(`{"key": "my-user"}`), headers, nil, ctx) resp := httptest.NewRecorder() - evaluateAllFeatureFlags(basictypes.JSClientSDK)(resp, req) + evaluateAllFeatureFlags(basictypes.JSClientSDK, ct.OptBase2Bytes{})(resp, req) assert.Equal(t, http.StatusServiceUnavailable, resp.Code) @@ -44,13 +45,42 @@ func TestReportFlagEvalFailsWithUninitializedClientAndStore(t *testing.T) { assert.JSONEq(t, `{"message":"Service not initialized"}`, string(b)) } +func TestReportFlagEvalRejectsOversizedBodyWhenLimitConfigured(t *testing.T) { + headers := make(http.Header) + headers.Set("Content-Type", "application/json") + ctx := testenv.NewTestEnvContext("", false, st.MakeStoreWithData(true)) + + maxBodySize, _ := ct.NewOptBase2BytesFromString("1KiB") + oversized := make([]byte, 1024*2) + for i := range oversized { + oversized[i] = 'a' + } + req := buildPreRoutedRequest("REPORT", oversized, headers, nil, ctx) + resp := httptest.NewRecorder() + evaluateAllFeatureFlags(basictypes.JSClientSDK, maxBodySize)(resp, req) + + assert.Equal(t, http.StatusRequestEntityTooLarge, resp.Code) +} + +func TestReportFlagEvalAllowsLargeBodyWhenNoLimitConfigured(t *testing.T) { + headers := make(http.Header) + headers.Set("Content-Type", "application/json") + ctx := testenv.NewTestEnvContext("", false, st.MakeStoreWithData(true)) + + req := buildPreRoutedRequest("REPORT", jsonhelpers.ToJSON(st.BasicUserForTestFlags), headers, nil, ctx) + resp := httptest.NewRecorder() + evaluateAllFeatureFlags(basictypes.JSClientSDK, ct.OptBase2Bytes{})(resp, req) + + assert.Equal(t, http.StatusOK, resp.Code) +} + func TestReportFlagEvalWorksWithUninitializedClientButInitializedStore(t *testing.T) { headers := make(http.Header) headers.Set("Content-Type", "application/json") ctx := testenv.NewTestEnvContext("", false, st.MakeStoreWithData(true)) req := buildPreRoutedRequest("REPORT", jsonhelpers.ToJSON(st.BasicUserForTestFlags), headers, nil, ctx) resp := httptest.NewRecorder() - evaluateAllFeatureFlags(basictypes.JSClientSDK)(resp, req) + evaluateAllFeatureFlags(basictypes.JSClientSDK, ct.OptBase2Bytes{})(resp, req) assert.Equal(t, http.StatusOK, resp.Code) diff --git a/relay/relay_routes.go b/relay/relay_routes.go index b845dcf7..2d169244 100644 --- a/relay/relay_routes.go +++ b/relay/relay_routes.go @@ -47,6 +47,7 @@ func (r *Relay) makeRouter() *mux.Router { mobileKeySelector := middleware.SelectEnvironmentByAuthorizationKey(basictypes.MobileSDK, environmentGetters) jsClientSelector := middleware.SelectEnvironmentByAuthorizationKey(basictypes.JSClientSDK, environmentGetters) offlineMode := r.config.OfflineMode.FileDataSource != "" + maxClientRequestBodySize := r.config.Main.MaxClientRequestBodySize // Client-side evaluation (for JS, not mobile) jsClientSideMiddlewareStack := func(subrouter *mux.Router) mux.MiddlewareFunc { @@ -65,10 +66,10 @@ func (r *Relay) makeRouter() *mux.Router { clientSideSdkEvalXRouter := router.PathPrefix("/sdk/evalx/{envId}/").Subrouter() clientSideSdkEvalXRouter.Use(jsClientSideMiddlewareStack(clientSideSdkEvalXRouter)) - clientSideSdkEvalXRouter.HandleFunc("/contexts/{context}", evaluateAllFeatureFlags(basictypes.JSClientSDK)).Methods("GET", "OPTIONS") - clientSideSdkEvalXRouter.HandleFunc("/context", evaluateAllFeatureFlags(basictypes.JSClientSDK)).Methods("REPORT", "OPTIONS") - clientSideSdkEvalXRouter.HandleFunc("/users/{context}", evaluateAllFeatureFlags(basictypes.JSClientSDK)).Methods("GET", "OPTIONS") - clientSideSdkEvalXRouter.HandleFunc("/user", evaluateAllFeatureFlags(basictypes.JSClientSDK)).Methods("REPORT", "OPTIONS") + clientSideSdkEvalXRouter.HandleFunc("/contexts/{context}", evaluateAllFeatureFlags(basictypes.JSClientSDK, maxClientRequestBodySize)).Methods("GET", "OPTIONS") + clientSideSdkEvalXRouter.HandleFunc("/context", evaluateAllFeatureFlags(basictypes.JSClientSDK, maxClientRequestBodySize)).Methods("REPORT", "OPTIONS") + clientSideSdkEvalXRouter.HandleFunc("/users/{context}", evaluateAllFeatureFlags(basictypes.JSClientSDK, maxClientRequestBodySize)).Methods("GET", "OPTIONS") + clientSideSdkEvalXRouter.HandleFunc("/user", evaluateAllFeatureFlags(basictypes.JSClientSDK, maxClientRequestBodySize)).Methods("REPORT", "OPTIONS") serverSideMiddlewareStack := middleware.Chain( sdkKeySelector, @@ -82,12 +83,12 @@ func (r *Relay) makeRouter() *mux.Router { // serverSideSdkRouter.Use(serverSideMiddlewareStack) serverSideEvalXRouter := serverSideSdkRouter.PathPrefix("/evalx/").Subrouter() - serverSideEvalXRouter.Handle("/contexts/{context}", serverSideMiddlewareStack(middleware.PollingRequestCount(http.HandlerFunc(evaluateAllFeatureFlags(basictypes.ServerSDK))))).Methods("GET") - serverSideEvalXRouter.Handle("/context", serverSideMiddlewareStack(middleware.PollingRequestCount(http.HandlerFunc(evaluateAllFeatureFlags(basictypes.ServerSDK))))).Methods("REPORT") + serverSideEvalXRouter.Handle("/contexts/{context}", serverSideMiddlewareStack(middleware.PollingRequestCount(http.HandlerFunc(evaluateAllFeatureFlags(basictypes.ServerSDK, maxClientRequestBodySize))))).Methods("GET") + serverSideEvalXRouter.Handle("/context", serverSideMiddlewareStack(middleware.PollingRequestCount(http.HandlerFunc(evaluateAllFeatureFlags(basictypes.ServerSDK, maxClientRequestBodySize))))).Methods("REPORT") // /users and /user are obsolete names for /contexts and /context, still used by some supported SDKs; the handler is // the same, because in both cases LD accepts any valid user *or* context JSON. - serverSideEvalXRouter.Handle("/users/{context}", serverSideMiddlewareStack(middleware.PollingRequestCount(http.HandlerFunc(evaluateAllFeatureFlags(basictypes.ServerSDK))))).Methods("GET") - serverSideEvalXRouter.Handle("/user", serverSideMiddlewareStack(middleware.PollingRequestCount(http.HandlerFunc(evaluateAllFeatureFlags(basictypes.ServerSDK))))).Methods("REPORT") + serverSideEvalXRouter.Handle("/users/{context}", serverSideMiddlewareStack(middleware.PollingRequestCount(http.HandlerFunc(evaluateAllFeatureFlags(basictypes.ServerSDK, maxClientRequestBodySize))))).Methods("GET") + serverSideEvalXRouter.Handle("/user", serverSideMiddlewareStack(middleware.PollingRequestCount(http.HandlerFunc(evaluateAllFeatureFlags(basictypes.ServerSDK, maxClientRequestBodySize))))).Methods("REPORT") // PHP SDK endpoints serverSideSdkRouter.Handle("/flags", serverSideMiddlewareStack(middleware.PollingRequestCount(http.HandlerFunc(pollAllFlagsHandler)))).Methods("GET") @@ -104,16 +105,16 @@ func (r *Relay) makeRouter() *mux.Router { msdkRouter.Use(mobileMiddlewareStack) msdkEvalXRouter := msdkRouter.PathPrefix("/evalx/").Subrouter() - msdkEvalXRouter.HandleFunc("/contexts/{context}", evaluateAllFeatureFlags(basictypes.MobileSDK)).Methods("GET") - msdkEvalXRouter.HandleFunc("/context", evaluateAllFeatureFlags(basictypes.MobileSDK)).Methods("REPORT") + msdkEvalXRouter.HandleFunc("/contexts/{context}", evaluateAllFeatureFlags(basictypes.MobileSDK, maxClientRequestBodySize)).Methods("GET") + msdkEvalXRouter.HandleFunc("/context", evaluateAllFeatureFlags(basictypes.MobileSDK, maxClientRequestBodySize)).Methods("REPORT") // /users and /user are obsolete names for /contexts and /context, still used by some supported SDKs; the handler is // the same, because in both cases LD accepts any valid user *or* context JSON. - msdkEvalXRouter.HandleFunc("/users/{context}", evaluateAllFeatureFlags(basictypes.MobileSDK)).Methods("GET") - msdkEvalXRouter.HandleFunc("/user", evaluateAllFeatureFlags(basictypes.MobileSDK)).Methods("REPORT") + msdkEvalXRouter.HandleFunc("/users/{context}", evaluateAllFeatureFlags(basictypes.MobileSDK, maxClientRequestBodySize)).Methods("GET") + msdkEvalXRouter.HandleFunc("/user", evaluateAllFeatureFlags(basictypes.MobileSDK, maxClientRequestBodySize)).Methods("REPORT") mobileStreamRouter := router.PathPrefix("/meval").Subrouter() mobileStreamRouter.Use(mobileMiddlewareStack, middleware.Streaming) - mobilePingWithUser := pingStreamHandlerWithContext(basictypes.MobileSDK, r.mobileStreamProvider) + mobilePingWithUser := pingStreamHandlerWithContext(basictypes.MobileSDK, maxClientRequestBodySize, r.mobileStreamProvider) mobileStreamRouter.Handle("", middleware.UsageActivityStreamMonitoring(metrics.MobilePlatformCategory, middleware.CountMobileConns(mobilePingWithUser))).Methods("REPORT") mobileStreamRouter.Handle("/{context}", middleware.UsageActivityStreamMonitoring(metrics.MobilePlatformCategory, middleware.CountMobileConns(mobilePingWithUser))).Methods("GET") @@ -121,7 +122,7 @@ func (r *Relay) makeRouter() *mux.Router { middleware.UsageActivityStreamMonitoring(metrics.MobilePlatformCategory, middleware.CountMobileConns(middleware.Streaming(pingStreamHandler(r.mobileStreamProvider)))))).Methods("GET") jsPing := pingStreamHandler(r.jsClientStreamProvider) - jsPingWithUser := pingStreamHandlerWithContext(basictypes.JSClientSDK, r.jsClientStreamProvider) + jsPingWithUser := pingStreamHandlerWithContext(basictypes.JSClientSDK, maxClientRequestBodySize, r.jsClientStreamProvider) clientSidePingRouter := router.PathPrefix("/ping/{envId}").Subrouter() clientSidePingRouter.Use(jsClientSideMiddlewareStack(clientSidePingRouter), middleware.Streaming) From dcea519fd6f6288c64cc2b3fef9765fae62a7ca8 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 12 Aug 2026 10:04:35 -0700 Subject: [PATCH 55/66] fix(concurrent-keys): reject SDK keys and mobile keys scoped to a view (#795) Makes the Relay Proxy reject SDK keys and mobile keys that are scoped to a view, so they never enter the accepted credential set. Filtering happens in envfactory.BuildAcceptedSet so a view-scoped key is absent from the credential lookup map entirely. An SDK presenting one gets a 401. --- internal/autoconfig/stream_manager.go | 13 +- internal/credential/accepted_set.go | 16 +- internal/credential/accepted_set_builder.go | 6 +- .../credential/accepted_set_builder_test.go | 6 +- internal/envfactory/env_params.go | 16 +- internal/envfactory/env_rep.go | 17 +- internal/envfactory/env_rep_test.go | 61 ++++++ internal/envfactory/reconcile_helper.go | 55 ++++-- internal/envfactory/reconcile_helper_test.go | 183 +++++++++++++++--- relay/autoconfig_actions.go | 22 ++- relay/filedata_actions.go | 26 +-- 11 files changed, 323 insertions(+), 98 deletions(-) diff --git a/internal/autoconfig/stream_manager.go b/internal/autoconfig/stream_manager.go index 1c77fe87..5c15b845 100644 --- a/internal/autoconfig/stream_manager.go +++ b/internal/autoconfig/stream_manager.go @@ -498,16 +498,11 @@ func (s *StreamManager) dispatchEnvAction(id config.EnvironmentID, rep envfactor } // validateCredentialPayload checks that an environment rep carries a structurally valid credential -// set. It is run at the stream parse boundary — before the rep's version is recorded via Upsert — -// mirroring how an unparseable event is handled by gotMalformedEvent. -// -// A malformed credential payload must preserve the previous accepted set and force a -// stream reconnect (RAC is one-way push with no NAK channel, so the reconnect is what makes the -// backend resend a fresh put). Validating here rather than after Upsert is essential: the version is -// not advanced, so the fresh put — which carries the same version — is not deduplicated away by the -// MessageReceiver. Any error from BuildAcceptedSet is a *MalformedCredentialSetError. +// set. It runs at the stream parse boundary, before the rep's version is recorded via Upsert: a +// malformed payload must not advance the version, or the backend's fresh put — which carries the same +// version — would be deduplicated away by the MessageReceiver. func (s *StreamManager) validateCredentialPayload(rep envfactory.EnvironmentRep) error { - _, err := envfactory.BuildAcceptedSet(rep.ToParams()) + _, _, err := envfactory.BuildAcceptedSet(rep.ToParams()) return err } diff --git a/internal/credential/accepted_set.go b/internal/credential/accepted_set.go index 24dee4f3..c3011929 100644 --- a/internal/credential/accepted_set.go +++ b/internal/credential/accepted_set.go @@ -1,7 +1,6 @@ package credential import ( - "errors" "fmt" "github.com/launchdarkly/ld-relay/v8/config" @@ -48,11 +47,6 @@ func (s AcceptedSet) hasMobileKey(key config.MobileKey) bool { return ok } -// errAcceptedSetMissingSDKKey is returned by AcceptedSetBuilder.Build when no SDK key was added. An -// environment must always have at least one SDK key (its anchor), so an empty set indicates a caller -// mistake rather than a benign edge case — surfacing it avoids a silent misconfiguration. -var errAcceptedSetMissingSDKKey = errors.New("accepted credential set must contain at least one SDK key") - // MalformedCredentialSetError is returned when a credential payload cannot produce a valid // AcceptedSet. This covers: // @@ -65,6 +59,9 @@ var errAcceptedSetMissingSDKKey = errors.New("accepted credential set must conta // would keep using the previous (possibly revoked) primary. (No mobile keys at all is valid.) // 4. An entry in sdkKeys[] or mobileKeys[] has an empty value — a credential that would be // accepted by relay but can never authenticate any SDK. +// 5. No SDK key survived at all, so the environment would have nothing to authenticate with. A +// payload reaches this by combining an undefined anchor with an sdkKeys[] array that is either +// empty or entirely made up of keys relay excludes, such as keys scoped to a view. // // Validation happens before Reconcile is called; Rotator.Reconcile trusts the set it is handed. // Because the error is raised before any state mutation, the environment's previous accepted set is @@ -86,6 +83,13 @@ func newMissingAnchorError() *MalformedCredentialSetError { return &MalformedCredentialSetError{msg: "malformed credential set: anchor SDK key is missing"} } +// newNoSDKKeysError returns a MalformedCredentialSetError for a set that ended up with no SDK key at +// all. The message describes the payload rather than the builder, because that is what an operator +// reading the log can act on. +func newNoSDKKeysError() *MalformedCredentialSetError { + return &MalformedCredentialSetError{msg: "malformed credential set: no usable SDK key in sdkKeys[]"} +} + // NewAnchorNotInSetError returns a MalformedCredentialSetError for a payload whose designated anchor // (sdkKey.value) is defined but not present in the sdkKeys[] array — a structural inconsistency. The // anchor value is a secret, so it is deliberately not included in the message. diff --git a/internal/credential/accepted_set_builder.go b/internal/credential/accepted_set_builder.go index 47136a8b..d6332749 100644 --- a/internal/credential/accepted_set_builder.go +++ b/internal/credential/accepted_set_builder.go @@ -89,13 +89,13 @@ func (b *AcceptedSetBuilder) WithEnvironmentID(id config.EnvironmentID) *Accepte return b } -// Build validates and returns the accumulated AcceptedSet. It returns errAcceptedSetMissingSDKKey if -// no SDK key was added, or a *MalformedCredentialSetError if no anchor was designated (via +// Build validates and returns the accumulated AcceptedSet. It returns a +// *MalformedCredentialSetError if no SDK key was added, or if no anchor was designated (via // WithAnchor). Because WithAnchor also adds the key, a designated anchor is always among the // accepted SDK keys. func (b *AcceptedSetBuilder) Build() (AcceptedSet, error) { if len(b.set.sdkKeys) == 0 { - return AcceptedSet{}, errAcceptedSetMissingSDKKey + return AcceptedSet{}, newNoSDKKeysError() } if !b.set.anchor.Defined() { return AcceptedSet{}, newMissingAnchorError() diff --git a/internal/credential/accepted_set_builder_test.go b/internal/credential/accepted_set_builder_test.go index 3f89f9c9..3b7b6be4 100644 --- a/internal/credential/accepted_set_builder_test.go +++ b/internal/credential/accepted_set_builder_test.go @@ -10,15 +10,15 @@ import ( ) func TestAcceptedSetBuilderValidation(t *testing.T) { - // No SDK key at all is a caller error. + // No SDK key at all is malformed: the environment would have nothing to authenticate with. + var malformed *MalformedCredentialSetError _, err := NewAcceptedSetBuilder(). WithMobileKey(MobileKeyParams{Value: "mob"}). WithEnvironmentID(config.EnvironmentID("env")). Build() - require.ErrorIs(t, err, errAcceptedSetMissingSDKKey) + require.ErrorAs(t, err, &malformed) // An SDK key with no designated anchor is malformed. - var malformed *MalformedCredentialSetError _, err = NewAcceptedSetBuilder().WithSDKKey(SDKKeyParams{Value: "sdk"}).Build() require.ErrorAs(t, err, &malformed) diff --git a/internal/envfactory/env_params.go b/internal/envfactory/env_params.go index 35acffcd..20d275b2 100644 --- a/internal/envfactory/env_params.go +++ b/internal/envfactory/env_params.go @@ -46,18 +46,22 @@ type EnvironmentParams struct { // AcceptedSDKKey is one entry in the accepted SDK key set for an environment. // Expiry is zero if the key is permanent. +// HasViews is true if the SDK key is associated with a view. type AcceptedSDKKey struct { - Key string - Value config.SDKKey - Expiry time.Time + Key string + Value config.SDKKey + Expiry time.Time + HasViews bool } // AcceptedMobileKey is one entry in the accepted mobile key set for an environment. // Expiry is zero if the key is permanent. +// HasViews is true if the mobile key is associated with a view. type AcceptedMobileKey struct { - Key string - Value config.MobileKey - Expiry time.Time + Key string + Value config.MobileKey + Expiry time.Time + HasViews bool } func (e EnvironmentParams) WithFilter(key config.FilterKey) EnvironmentParams { diff --git a/internal/envfactory/env_rep.go b/internal/envfactory/env_rep.go index d83f7ec0..0b150f72 100644 --- a/internal/envfactory/env_rep.go +++ b/internal/envfactory/env_rep.go @@ -105,9 +105,10 @@ type ExpiringKeyRep struct { // Key is the human-readable identifier (non-secret, e.g. "default-sdk"); Value is // the credential secret (e.g. "sdk-xxxx-..."). See the EnvironmentRep TERMINOLOGY comment. type ConcurrentKeyRep struct { - Key string `json:"key"` - Value string `json:"value"` - Expiry *int64 `json:"expiry,omitempty"` // Unix-ms; nil = permanent + Key string `json:"key"` + Value string `json:"value"` + Expiry *int64 `json:"expiry,omitempty"` // Unix-ms; nil = permanent + HasViews bool `json:"hasViews"` } func ToTime(millisecondTime ldtime.UnixMillisecondTime) time.Time { @@ -135,8 +136,9 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { params.AcceptedSDKKeys = make([]AcceptedSDKKey, 0, len(r.SDKKeys)) for _, k := range r.SDKKeys { entry := AcceptedSDKKey{ - Key: k.Key, - Value: config.SDKKey(k.Value), + Key: k.Key, + Value: config.SDKKey(k.Value), + HasViews: k.HasViews, } if k.Expiry != nil { entry.Expiry = time.UnixMilli(*k.Expiry) @@ -162,8 +164,9 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { params.AcceptedMobileKeys = make([]AcceptedMobileKey, 0, len(r.MobileKeys)) for _, k := range r.MobileKeys { entry := AcceptedMobileKey{ - Key: k.Key, - Value: config.MobileKey(k.Value), + Key: k.Key, + Value: config.MobileKey(k.Value), + HasViews: k.HasViews, } if k.Expiry != nil { entry.Expiry = time.UnixMilli(*k.Expiry) diff --git a/internal/envfactory/env_rep_test.go b/internal/envfactory/env_rep_test.go index 2499b045..6baaea75 100644 --- a/internal/envfactory/env_rep_test.go +++ b/internal/envfactory/env_rep_test.go @@ -159,6 +159,67 @@ func TestEnvironmentRepNewFormatWithArrays(t *testing.T) { assert.Equal(t, AcceptedMobileKey{Key: "mob-key-1", Value: config.MobileKey("mob-f41c")}, params.AcceptedMobileKeys[0]) } +// TestEnvironmentRepViewScopedKeys pins the hasViews wire contract on both arrays: it decodes onto +// ConcurrentKeyRep and is carried through ToParams onto the accepted entries, where BuildAcceptedSet +// consumes it. +// +// The absent-field case is the important one. hasViews is a plain bool, so an entry that omits it — +// every entry a backend that predates the field emits — decodes to false and is treated as not +// view-scoped. An explicit false is indistinguishable from absent, which is the intent: there is no +// third state. +func TestEnvironmentRepViewScopedKeys(t *testing.T) { + jsonStr := `{ + "envID": "68e5179e8307e4099c277e2a", + "envKey": "production", + "envName": "Production", + "mobKey": "mob-primary", + "projKey": "my-project", + "projName": "My Project", + "sdkKey": { "value": "sdk-anchor" }, + "sdkKeys": [ + { "key": "default-sdk", "value": "sdk-anchor" }, + { "key": "service-a", "value": "sdk-service-a", "hasViews": false }, + { "key": "view-scoped", "value": "sdk-viewy", "hasViews": true } + ], + "mobileKeys": [ + { "key": "default-mob", "value": "mob-primary" }, + { "key": "view-scoped-mob", "value": "mob-viewy", "hasViews": true } + ], + "version": 26 + }` + + var rep EnvironmentRep + require.NoError(t, json.Unmarshal([]byte(jsonStr), &rep)) + + require.Len(t, rep.SDKKeys, 3) + assert.False(t, rep.SDKKeys[0].HasViews, "an absent hasViews must decode to false") + assert.False(t, rep.SDKKeys[1].HasViews) + assert.True(t, rep.SDKKeys[2].HasViews) + + require.Len(t, rep.MobileKeys, 2) + assert.False(t, rep.MobileKeys[0].HasViews) + assert.True(t, rep.MobileKeys[1].HasViews) + + params := rep.ToParams() + + require.Len(t, params.AcceptedSDKKeys, 3) + assert.Equal(t, AcceptedSDKKey{Key: "default-sdk", Value: config.SDKKey("sdk-anchor")}, params.AcceptedSDKKeys[0]) + assert.Equal(t, AcceptedSDKKey{Key: "service-a", Value: config.SDKKey("sdk-service-a")}, params.AcceptedSDKKeys[1]) + assert.Equal(t, AcceptedSDKKey{ + Key: "view-scoped", + Value: config.SDKKey("sdk-viewy"), + HasViews: true, + }, params.AcceptedSDKKeys[2]) + + require.Len(t, params.AcceptedMobileKeys, 2) + assert.Equal(t, AcceptedMobileKey{Key: "default-mob", Value: config.MobileKey("mob-primary")}, params.AcceptedMobileKeys[0]) + assert.Equal(t, AcceptedMobileKey{ + Key: "view-scoped-mob", + Value: config.MobileKey("mob-viewy"), + HasViews: true, + }, params.AcceptedMobileKeys[1]) +} + // TestEnvironmentRepOldFormatNoArrays verifies that an old-format payload (singular sdkKey/mobKey // only, no sdkKeys/mobileKeys arrays) is normalized by ToParams() into a consistent accepted set. // The wire rep's SDKKeys/MobileKeys remain nil, but params.AcceptedSDKKeys/AcceptedMobileKeys are diff --git a/internal/envfactory/reconcile_helper.go b/internal/envfactory/reconcile_helper.go index 4a5f418e..5052bcb7 100644 --- a/internal/envfactory/reconcile_helper.go +++ b/internal/envfactory/reconcile_helper.go @@ -20,33 +20,44 @@ import ( // The builder de-duplicates by value, so an anchor or primary mobile key that also appears in its // array is added only once. // -// A *credential.MalformedCredentialSetError is returned (with an empty AcceptedSet) for a -// structurally malformed payload: an undefined anchor (params.SDKKey not set), a defined anchor that -// is absent from params.AcceptedSDKKeys, a defined primary mobile key (params.MobileKey) that is -// absent from params.AcceptedMobileKeys, a non-empty params.AcceptedMobileKeys with no designated -// primary (params.MobileKey undefined), or an array entry with an empty value. The caller must -// preserve the previous accepted state and, for RAC handlers, reconnect the stream with jitter to -// force a fresh put. This is the single home for the anchor invariant. -func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, error) { +// An error is returned (with an empty AcceptedSet) for a structurally malformed payload: an undefined +// anchor (params.SDKKey not set), a defined anchor that is absent from params.AcceptedSDKKeys, a +// defined primary mobile key (params.MobileKey) that is absent from params.AcceptedMobileKeys, a +// non-empty params.AcceptedMobileKeys with no designated primary (params.MobileKey undefined), an +// array entry with an empty value, or no usable SDK key at all. The caller must preserve the previous +// accepted state and, for RAC handlers, reconnect the stream with jitter to force a fresh put. This is +// the single home for the anchor invariant. +// +// Keys scoped to a view are filtered out here rather than at authentication time, making this the +// single funnel for both RAC and the offline archive. The second return value names the keys that were +// dropped, so callers can log what they lost. An SDK presenting one of them gets a 401: the key is +// simply absent from the lookup map. +func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, []string, error) { anchor := params.SDKKey b := credential.NewAcceptedSetBuilder().WithEnvironmentID(params.EnvID) + var rejected []string // Add every accepted SDK key, designating the anchor as we encounter it. WithAnchor both adds and // designates, and forces the anchor permanent — so a payload that (wrongly) carries an expiry on // the anchor's own entry cannot demote it. An undefined anchor never matches a (defined) array - // value, so it is never designated and Build returns a *MalformedCredentialSetError. + // value, so it is never designated and Build rejects the payload. // // Entries with an empty value are structurally malformed: relay would silently accept them but // they can never authenticate any SDK. Reject loudly rather than produce a credential-short env. anchorInArray := false for _, k := range params.AcceptedSDKKeys { if !k.Value.Defined() { - return credential.AcceptedSet{}, credential.NewEmptyCredentialError("sdkKeys", k.Key) + return credential.AcceptedSet{}, nil, credential.NewEmptyCredentialError("sdkKeys", k.Key) } - if k.Value == anchor { + switch { + // A marker on the anchor's own entry is disregarded: dropping the designated key would take the + // whole environment down, and the backend forbids views on a default key in the first place. + case k.Value == anchor: anchorInArray = true b.WithAnchor(credential.SDKKeyParams{Value: k.Value, Key: util.PtrOrNil(k.Key)}) - } else { + case k.HasViews: + rejected = append(rejected, k.Key) + default: b.WithSDKKey(credential.SDKKeyParams{Value: k.Value, Key: util.PtrOrNil(k.Key), Expiry: util.PtrOrNil(k.Expiry)}) } } @@ -55,7 +66,7 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, error) // synthesizes it into the array for old-format payloads). A defined anchor absent from the array is // a structurally malformed payload — reject it. if anchor.Defined() && !anchorInArray { - return credential.AcceptedSet{}, credential.NewAnchorNotInSetError() + return credential.AcceptedSet{}, nil, credential.NewAnchorNotInSetError() } // Add every accepted mobile key, designating the primary as we encounter it. Like the anchor, @@ -64,12 +75,16 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, error) primaryMobileInArray := false for _, k := range params.AcceptedMobileKeys { if !k.Value.Defined() { - return credential.AcceptedSet{}, credential.NewEmptyCredentialError("mobileKeys", k.Key) + return credential.AcceptedSet{}, nil, credential.NewEmptyCredentialError("mobileKeys", k.Key) } - if k.Value == params.MobileKey { + switch { + // Like the anchor, a marker on the primary's own entry is disregarded rather than honored. + case k.Value == params.MobileKey: primaryMobileInArray = true b.WithPrimaryMobileKey(credential.MobileKeyParams{Value: k.Value, Key: util.PtrOrNil(k.Key)}) - } else { + case k.HasViews: + rejected = append(rejected, k.Key) + default: b.WithMobileKey(credential.MobileKeyParams{Value: k.Value, Key: util.PtrOrNil(k.Key), Expiry: util.PtrOrNil(k.Expiry)}) } } @@ -79,7 +94,7 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, error) // without this guard the primary would be silently left undesignated, clearing it on reconcile and // breaking event forwarding. (An undefined mobKey is valid — a server-side-only environment.) if params.MobileKey.Defined() && !primaryMobileInArray { - return credential.AcceptedSet{}, credential.NewPrimaryMobileKeyNotInSetError() + return credential.AcceptedSet{}, nil, credential.NewPrimaryMobileKeyNotInSetError() } // A non-empty mobileKeys[] with no designated primary (undefined mobKey) is malformed: the reconcile @@ -89,12 +104,12 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, error) // server-side-only environment. Old-format payloads synthesize the array from mobKey only, so an // undefined mobKey yields an empty array and is unaffected.) if len(params.AcceptedMobileKeys) > 0 && !params.MobileKey.Defined() { - return credential.AcceptedSet{}, credential.NewPrimaryMobileKeyMissingError() + return credential.AcceptedSet{}, nil, credential.NewPrimaryMobileKeyMissingError() } set, err := b.Build() if err != nil { - return credential.AcceptedSet{}, err + return credential.AcceptedSet{}, nil, err } - return set, nil + return set, rejected, nil } diff --git a/internal/envfactory/reconcile_helper_test.go b/internal/envfactory/reconcile_helper_test.go index df87d178..7ec90403 100644 --- a/internal/envfactory/reconcile_helper_test.go +++ b/internal/envfactory/reconcile_helper_test.go @@ -50,7 +50,7 @@ func TestBuildAcceptedSet_HappyPath(t *testing.T) { []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, "mob-primary", ) - set, err := BuildAcceptedSet(params) + set, _, err := BuildAcceptedSet(params) require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). @@ -72,7 +72,7 @@ func TestBuildAcceptedSet_MultipleKeys(t *testing.T) { }, "mob-primary", ) - set, err := BuildAcceptedSet(params) + set, _, err := BuildAcceptedSet(params) require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). @@ -99,8 +99,8 @@ func TestBuildAcceptedSet_Rename(t *testing.T) { "mob-primary", ) - setOld, errOld := BuildAcceptedSet(paramsOldName) - setNew, errNew := BuildAcceptedSet(paramsNewName) + setOld, _, errOld := BuildAcceptedSet(paramsOldName) + setNew, _, errNew := BuildAcceptedSet(paramsNewName) require.NoError(t, errOld) require.NoError(t, errNew) @@ -143,8 +143,8 @@ func TestBuildAcceptedSet_Deexpiry(t *testing.T) { "mob-primary", ) - setWithExpiry, errWithExpiry := BuildAcceptedSet(paramsWithExpiry) - setNoExpiry, errNoExpiry := BuildAcceptedSet(paramsNoExpiry) + setWithExpiry, _, errWithExpiry := BuildAcceptedSet(paramsWithExpiry) + setNoExpiry, _, errNoExpiry := BuildAcceptedSet(paramsNoExpiry) require.NoError(t, errWithExpiry) require.NoError(t, errNoExpiry) @@ -173,7 +173,7 @@ func TestBuildAcceptedSet_AnchorNotInArray(t *testing.T) { }, "mob-primary", ) - _, err := BuildAcceptedSet(params) + _, _, err := BuildAcceptedSet(params) require.Error(t, err) var malformed *credential.MalformedCredentialSetError @@ -194,7 +194,7 @@ func TestBuildAcceptedSet_PrimaryMobileNotInArray(t *testing.T) { {Key: "other", Value: "mob-other"}, // ...but NOT in the array }, } - _, err := BuildAcceptedSet(params) + _, _, err := BuildAcceptedSet(params) require.Error(t, err) var malformed *credential.MalformedCredentialSetError @@ -211,7 +211,7 @@ func TestBuildAcceptedSet_NoMobileKey(t *testing.T) { SDKKey: SDKKeyRep{Value: config.SDKKey("sdk-anchor")}, // no MobKey, no MobileKeys } - set, err := BuildAcceptedSet(rep.ToParams()) + set, _, err := BuildAcceptedSet(rep.ToParams()) require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). @@ -235,7 +235,7 @@ func TestBuildAcceptedSet_MobileKeysWithoutPrimary(t *testing.T) { {Key: "mob-1", Value: "mob-primary"}, // ...but the array is non-empty }, } - _, err := BuildAcceptedSet(params) + _, _, err := BuildAcceptedSet(params) require.Error(t, err) var malformed *credential.MalformedCredentialSetError @@ -254,7 +254,7 @@ func TestBuildAcceptedSet_EmptyMobileArrayValid(t *testing.T) { AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, AcceptedMobileKeys: []AcceptedMobileKey{}, // empty } - set, err := BuildAcceptedSet(params) + set, _, err := BuildAcceptedSet(params) require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). @@ -273,7 +273,7 @@ func TestBuildAcceptedSet_AnchorUndefined(t *testing.T) { }, "mob-primary", ) - _, err := BuildAcceptedSet(params) + _, _, err := BuildAcceptedSet(params) require.Error(t, err) var malformed *credential.MalformedCredentialSetError @@ -281,16 +281,32 @@ func TestBuildAcceptedSet_AnchorUndefined(t *testing.T) { assert.Contains(t, malformed.Error(), "anchor SDK key is missing") } -// TestBuildAcceptedSet_NoSDKKeys verifies that when neither an anchor nor any array SDK keys are -// present, Build returns an error (the set has no SDK key at all). +// TestBuildAcceptedSet_NoSDKKeys verifies that when no SDK key survives, the payload is rejected as +// malformed. Two shapes reach this, both requiring an undefined anchor: an empty array, and an array +// whose every entry is filtered out for being scoped to a view. The second is why the case is worth +// pinning by error type — a filtered-to-empty array is a payload problem, not a caller mistake. func TestBuildAcceptedSet_NoSDKKeys(t *testing.T) { - params := EnvironmentParams{ - SDKKey: "", // undefined anchor - AcceptedSDKKeys: []AcceptedSDKKey{}, - AcceptedMobileKeys: []AcceptedMobileKey{}, + tests := []struct { + name string + sdkKeys []AcceptedSDKKey + }{ + {"empty array", []AcceptedSDKKey{}}, + {"every entry view-scoped", []AcceptedSDKKey{{Key: "view-sdk", Value: "sdk-viewy", HasViews: true}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + params := EnvironmentParams{ + SDKKey: "", // undefined anchor + AcceptedSDKKeys: tt.sdkKeys, + AcceptedMobileKeys: []AcceptedMobileKey{}, + } + _, _, err := BuildAcceptedSet(params) + + require.Error(t, err, "a set with no SDK key at all must be rejected") + var malformed *credential.MalformedCredentialSetError + require.ErrorAs(t, err, &malformed, "every rejection from BuildAcceptedSet is a malformed-payload error") + }) } - _, err := BuildAcceptedSet(params) - require.Error(t, err, "a set with no SDK key at all must be rejected") } // TestBuildAcceptedSet_MixedUpdate verifies add + re-anchor + remove in a single params update @@ -312,7 +328,7 @@ func TestBuildAcceptedSet_MixedUpdate(t *testing.T) { }, "mob-primary", ) - set, err := BuildAcceptedSet(params) + set, _, err := BuildAcceptedSet(params) require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). @@ -336,7 +352,7 @@ func TestBuildAcceptedSet_AnchorNeverExpiring(t *testing.T) { }, "mob-primary", ) - set, err := BuildAcceptedSet(params) + set, _, err := BuildAcceptedSet(params) require.NoError(t, err) // Anchor is permanent (WithAnchor), not expiring — identical to a payload with no anchor expiry. @@ -362,7 +378,7 @@ func TestBuildAcceptedSet_MultipleMobileKeys(t *testing.T) { {Key: "mob-2", Value: "mob-secondary"}, }, } - set, err := BuildAcceptedSet(params) + set, _, err := BuildAcceptedSet(params) require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). @@ -388,7 +404,7 @@ func TestBuildAcceptedSet_ExpiringMobileKey(t *testing.T) { {Key: "mob-old", Value: "mob-old", Expiry: expiry1}, // expiring }, } - set, err := BuildAcceptedSet(params) + set, _, err := BuildAcceptedSet(params) require.NoError(t, err) expected := mustBuild(t, credential.NewAcceptedSetBuilder(). @@ -398,3 +414,122 @@ func TestBuildAcceptedSet_ExpiringMobileKey(t *testing.T) { WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary", Key: util.PtrOrNil("mob-1")})) assert.Equal(t, expected, set, "expiring mobile key must land as an expiring key in the set") } + +// TestBuildAcceptedSet_ViewScopedKeys covers the ingestion filter across both arrays. A key scoped to a +// view may only see a subset of the environment's flags; relay serves the whole environment payload, so +// admitting one would silently over-deliver. Such a key is therefore never added to the set, and an SDK +// presenting it is rejected because the credential is simply absent from the lookup map. The dropped +// keys are returned to the caller so it can WARN. +func TestBuildAcceptedSet_ViewScopedKeys(t *testing.T) { + const ( + anchor = config.SDKKey("sdk-anchor") + primary = config.MobileKey("mob-primary") + ) + + // The four entries every case starts from; individual cases add view-scoped entries alongside them. + anchorEntry := AcceptedSDKKey{Key: "default-sdk", Value: anchor} + extraSDK := AcceptedSDKKey{Key: "service-a", Value: "sdk-service-a"} + primaryEntry := AcceptedMobileKey{Key: "default-mob", Value: primary} + extraMob := AcceptedMobileKey{Key: "mob-extra", Value: "mob-extra"} + + // base is the set with only the two designated keys; cases add whatever survived the filter. + base := func() *credential.AcceptedSetBuilder { + return credential.NewAcceptedSetBuilder(). + WithEnvironmentID("env-abc"). + WithAnchor(credential.SDKKeyParams{Value: anchor, Key: util.PtrOrNil("default-sdk")}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: primary, Key: util.PtrOrNil("default-mob")}) + } + acceptedExtraSDK := credential.SDKKeyParams{Value: extraSDK.Value, Key: util.PtrOrNil(extraSDK.Key)} + acceptedExtraMob := credential.MobileKeyParams{Value: extraMob.Value, Key: util.PtrOrNil(extraMob.Key)} + + // baseWithExtras is base plus both non-designated keys — the expectation for every case where the + // filter drops nothing that was going to be accepted anyway. + baseWithExtras := func() *credential.AcceptedSetBuilder { + return base().WithSDKKey(acceptedExtraSDK).WithMobileKey(acceptedExtraMob) + } + + tests := []struct { + name string + sdkKeys []AcceptedSDKKey + mobileKeys []AcceptedMobileKey + wantSet func() *credential.AcceptedSetBuilder + wantRejected []string + }{ + { + // Baseline: nothing view-scoped behaves exactly as it did before the field existed. + name: "no view-scoped keys", + sdkKeys: []AcceptedSDKKey{anchorEntry, extraSDK}, + mobileKeys: []AcceptedMobileKey{primaryEntry, extraMob}, + wantSet: baseWithExtras, + }, + { + name: "view-scoped non-anchor SDK key is excluded", + sdkKeys: []AcceptedSDKKey{anchorEntry, extraSDK, {Key: "view-sdk", Value: "sdk-viewy", HasViews: true}}, + mobileKeys: []AcceptedMobileKey{primaryEntry, extraMob}, + wantSet: baseWithExtras, + wantRejected: []string{"view-sdk"}, + }, + { + name: "view-scoped non-primary mobile key is excluded", + sdkKeys: []AcceptedSDKKey{anchorEntry, extraSDK}, + mobileKeys: []AcceptedMobileKey{primaryEntry, extraMob, {Key: "view-mob", Value: "mob-viewy", HasViews: true}}, + wantSet: baseWithExtras, + wantRejected: []string{"view-mob"}, + }, + { + // Both arrays filter in one pass; SDK keys are walked first, hence the order. + name: "view-scoped keys in both arrays are excluded", + sdkKeys: []AcceptedSDKKey{anchorEntry, {Key: "view-sdk", Value: "sdk-viewy", HasViews: true}}, + mobileKeys: []AcceptedMobileKey{primaryEntry, {Key: "view-mob", Value: "mob-viewy", HasViews: true}}, + wantSet: base, + // Every non-designated key is view-scoped, so only the anchor and primary survive. + wantRejected: []string{"view-sdk", "view-mob"}, + }, + { + // A view-scoped key is dropped outright rather than being admitted as an expiring key — + // the marker takes precedence over expiry handling. + name: "view-scoped key carrying an expiry is still excluded", + sdkKeys: []AcceptedSDKKey{anchorEntry, {Key: "view-sdk", Value: "sdk-viewy", Expiry: expiry1, HasViews: true}}, + mobileKeys: []AcceptedMobileKey{primaryEntry}, + wantSet: base, + wantRejected: []string{"view-sdk"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + params := EnvironmentParams{ + EnvID: "env-abc", + SDKKey: anchor, + MobileKey: primary, + AcceptedSDKKeys: tt.sdkKeys, + AcceptedMobileKeys: tt.mobileKeys, + } + + set, rejected, err := BuildAcceptedSet(params) + + // A view-scoped key is filtered, never fatal — the environment always keeps operating. + require.NoError(t, err) + assert.Equal(t, mustBuild(t, tt.wantSet()), set) + assert.Equal(t, tt.wantRejected, rejected) + }) + } +} + +// TestBuildAcceptedSet_ViewScopedKeysEmptyOnError verifies that the rejected list is empty whenever an +// error is returned. The caller discards the whole payload and preserves its previous credentials in +// that case, so reporting keys it did not act on would produce a misleading WARN. +func TestBuildAcceptedSet_ViewScopedKeysEmptyOnError(t *testing.T) { + // The anchor is absent from sdkKeys[] — malformed — and a view-scoped entry is present alongside. + params := EnvironmentParams{ + EnvID: "env-abc", + SDKKey: "sdk-anchor", + AcceptedSDKKeys: []AcceptedSDKKey{{Key: "view-sdk", Value: "sdk-viewy", HasViews: true}}, + AcceptedMobileKeys: []AcceptedMobileKey{}, + } + + _, rejected, err := BuildAcceptedSet(params) + + require.Error(t, err) + assert.Empty(t, rejected) +} diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index c59c4a1a..54df1c3e 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -1,6 +1,10 @@ package relay import ( + "strings" + + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/ld-relay/v8/config" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" @@ -11,8 +15,20 @@ const ( logMsgAutoConfUpdateUnknownEnv = "Got auto-configuration update for environment %q but did not have previous configuration - will add" logMsgAutoConfDeleteUnknownEnv = "Got auto-configuration delete message for environment %s but did not have previous configuration - ignoring" logMsgAutoConfReceivedAllEnvironments = "Finished processing auto-configuration data" + + logMsgViewScopedKeysRejected = "Environment %q: rejecting credentials scoped to a view: %s." + + " The Relay Proxy serves the entire environment payload and cannot filter it to a view," + + " so SDKs presenting these credentials will be denied." ) +// logViewScopedKeys reports the view-scoped credentials that BuildAcceptedSet filtered out of a +// payload. This logs once per payload that actually reaches a handler. +func logViewScopedKeys(loggers ldlog.Loggers, envName string, rejected []string) { + if len(rejected) > 0 { + loggers.Warnf(logMsgViewScopedKeysRejected, envName, strings.Join(rejected, ", ")) + } +} + // relayAutoConfigActions is an implementation of the autoconfig.MessageHandler interface. The low-level // autoconfig.StreamManager component, which manages the configuration stream protocol, will call the // interface methods on this object to let us know when environments have been added or changed. @@ -32,11 +48,12 @@ func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentPar return } - set, buildErr := envfactory.BuildAcceptedSet(params) + set, rejected, buildErr := envfactory.BuildAcceptedSet(params) if buildErr != nil { a.r.loggers.Errorf(logMsgAutoConfEnvInitError, params.Identifiers.GetDisplayName(), buildErr) return } + logViewScopedKeys(a.r.loggers, params.Identifiers.GetDisplayName(), rejected) env.ReconcileCredentials(set) } @@ -51,7 +68,7 @@ func (a *relayAutoConfigActions) UpdateEnvironment(params envfactory.Environment env.SetTTL(params.TTL) env.SetSecureMode(params.SecureMode) - set, buildErr := envfactory.BuildAcceptedSet(params) + set, rejected, buildErr := envfactory.BuildAcceptedSet(params) if buildErr != nil { // Credential payloads are validated at the stream parse boundary (see StreamManager) before // being dispatched here, so a malformed set should not reach this point. Log defensively and @@ -59,6 +76,7 @@ func (a *relayAutoConfigActions) UpdateEnvironment(params envfactory.Environment a.r.loggers.Errorf(logMsgAutoConfEnvInitError, params.Identifiers.GetDisplayName(), buildErr) return } + logViewScopedKeys(a.r.loggers, params.Identifiers.GetDisplayName(), rejected) env.ReconcileCredentials(set) } diff --git a/relay/filedata_actions.go b/relay/filedata_actions.go index 0ce77763..27008d73 100644 --- a/relay/filedata_actions.go +++ b/relay/filedata_actions.go @@ -1,11 +1,8 @@ package relay import ( - "errors" "time" - "github.com/launchdarkly/ld-relay/v8/internal/credential" - "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" @@ -23,6 +20,8 @@ const ( logMsgOfflineEnvTimeoutError = "Unable to initialize offline environment %q: timed out waiting for client creation" logMsgInternalErrorUpdatedEnvNotFound = "Unexpected error in file data processing: environment ID %s not found when updating" logMsgInternalErrorNoUpdatesForEnv = "Unexpected error in file data processing: environment ID %s not found in envUpdates" + + logMsgOfflineMalformedPayload = "Malformed credential payload for offline environment %q — preserving previous credentials: %s" ) // relayFileDataActions is an implementation of the filedata.UpdateHandler interface. The low-level @@ -58,17 +57,13 @@ func (a *relayFileDataActions) AddEnvironment(ae filedata.ArchiveEnvironment) { return } - set, buildErr := envfactory.BuildAcceptedSet(ae.Params) + set, rejected, buildErr := envfactory.BuildAcceptedSet(ae.Params) if buildErr != nil { - var malformed *credential.MalformedCredentialSetError - if errors.As(buildErr, &malformed) { - a.r.loggers.Errorf("Malformed credential payload for offline environment %q — preserving previous credentials: %s", ae.Params.Identifiers.GetDisplayName(), buildErr) - } else { - a.r.loggers.Errorf(logMsgAutoConfEnvInitError, ae.Params.Identifiers.GetDisplayName(), buildErr) - } + a.r.loggers.Errorf(logMsgOfflineMalformedPayload, ae.Params.Identifiers.GetDisplayName(), buildErr) // No reconnect for offline mode: preserve previous state (env was just created with // the singular sdkKey from envConfig) and wait for the next archive reload. } else { + logViewScopedKeys(a.r.loggers, ae.Params.Identifiers.GetDisplayName(), rejected) env.ReconcileCredentials(set) } @@ -101,17 +96,12 @@ func (a *relayFileDataActions) UpdateEnvironment(ae filedata.ArchiveEnvironment) env.SetTTL(ae.Params.TTL) env.SetSecureMode(ae.Params.SecureMode) - set, buildErr := envfactory.BuildAcceptedSet(ae.Params) + set, rejected, buildErr := envfactory.BuildAcceptedSet(ae.Params) if buildErr != nil { - var malformed *credential.MalformedCredentialSetError - if errors.As(buildErr, &malformed) { - a.r.loggers.Errorf("Malformed credential payload for offline environment %q — preserving previous credentials: %s", ae.Params.Identifiers.GetDisplayName(), buildErr) - } else { - // The environment was found above; this is a credential-build failure, not a missing env. - a.r.loggers.Errorf(logMsgAutoConfEnvInitError, ae.Params.Identifiers.GetDisplayName(), buildErr) - } + a.r.loggers.Errorf(logMsgOfflineMalformedPayload, ae.Params.Identifiers.GetDisplayName(), buildErr) // Preserve previous credentials; no reconnect (offline path has no live stream). } else { + logViewScopedKeys(a.r.loggers, ae.Params.Identifiers.GetDisplayName(), rejected) env.ReconcileCredentials(set) } From 44742860198d659e30bc54c8f051ae38768244ec Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 12 Aug 2026 10:05:31 -0700 Subject: [PATCH 56/66] test(concurrent-keys): integration coverage for view-scoped key rejection (#796) --- relay/concurrent_keys_views_test.go | 361 ++++++++++++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 relay/concurrent_keys_views_test.go diff --git a/relay/concurrent_keys_views_test.go b/relay/concurrent_keys_views_test.go new file mode 100644 index 00000000..dbf0599a --- /dev/null +++ b/relay/concurrent_keys_views_test.go @@ -0,0 +1,361 @@ +package relay + +// Ingestion-time rejection of credentials scoped to a view (payload filtering). +// +// A view-scoped key may only see a subset of its environment's flags. The Relay Proxy serves the whole +// environment payload and has no view support, so accepting one would silently over-deliver every flag +// in the environment to an SDK entitled to a subset. These tests pin that such a key never reaches the +// accepted set — from either source — while everything around it keeps working. +// +// They mirror the four cases in concurrent_keys_auth_test.go and reuse its harnesses and fixtures +// (multiKeyEnvRep / multiKeyArchiveEnv, assertSDKEndpointsAvailability, awaitClient, awaitStreamClosed). +// The anchor and primary mobile key are the deliberate exception: the marker is ignored on them rather +// than taking the environment down, which is what TestConcurrentKeys*_ViewScopedMarkerOnDesignatedKeys* +// covers. + +import ( + "encoding/json" + "net/http" + "regexp" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/api" + "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/configsource" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + "github.com/launchdarkly/eventsource" + "github.com/launchdarkly/go-configtypes" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + // A non-anchor SDK key and a non-primary mobile key, each scoped to a view. + viewScopedSDKKey = config.SDKKey("sdk-view-scoped") + viewScopedMobileKey = config.MobileKey("mob-view-scoped") + + // The wire identifiers of those two entries. Non-secret, and what the WARN names. + viewScopedSDKID = "view-scoped-sdk" + viewScopedMobID = "view-scoped-mob" +) + +// The standard two-entry arrays plus a third entry scoped to a view, in each of the four shapes the +// two harnesses need (wire reps for RAC, accepted-key params for the offline archive). + +func viewScopedSDKKeyReps() []envfactory.ConcurrentKeyRep { + return append(defaultSDKKeyReps(), + envfactory.ConcurrentKeyRep{Key: viewScopedSDKID, Value: string(viewScopedSDKKey), HasViews: true}) +} + +func viewScopedMobileKeyReps() []envfactory.ConcurrentKeyRep { + return append(defaultMobileKeyReps(), + envfactory.ConcurrentKeyRep{Key: viewScopedMobID, Value: string(viewScopedMobileKey), HasViews: true}) +} + +func viewScopedAcceptedSDKKeys() []envfactory.AcceptedSDKKey { + return append(defaultAcceptedSDKKeys(), + envfactory.AcceptedSDKKey{Key: viewScopedSDKID, Value: viewScopedSDKKey, HasViews: true}) +} + +func viewScopedAcceptedMobileKeys() []envfactory.AcceptedMobileKey { + return append(defaultAcceptedMobileKeys(), + envfactory.AcceptedMobileKey{Key: viewScopedMobID, Value: viewScopedMobileKey, HasViews: true}) +} + +// assertViewScopedKeysAbsentFromStatus verifies the /status sdkKeys[]/mobileKeys[] arrays do not list +// the view-scoped credentials. Those arrays are rendered straight from the accepted set, so this is the +// externally-visible proof that a filtered key carries no state anywhere in the environment. +func assertViewScopedKeysAbsentFromStatus(t *testing.T, relay *Relay) { + t.Helper() + req, _ := http.NewRequest("GET", "/status", nil) + result, body := sharedtest.DoRequest(req, relay) + require.Equal(t, http.StatusOK, result.StatusCode) + + var status api.StatusRep + require.NoError(t, json.Unmarshal(body, &status)) + require.Len(t, status.Environments, 1) + var envStatus api.EnvironmentStatusRep + for _, e := range status.Environments { + envStatus = e + } + + assert.Nil(t, findSDKKeyStatus(envStatus.SDKKeys, sdks.ObscureKey(string(viewScopedSDKKey))), + "a view-scoped SDK key must not appear in the status sdkKeys[] array") + assert.Nil(t, findSDKKeyStatus(envStatus.MobileKeys, sdks.ObscureKey(string(viewScopedMobileKey))), + "a view-scoped mobile key must not appear in the status mobileKeys[] array") + + // The keys that were accepted are still all there — the filter is surgical, not a blanket drop. + assert.NotNil(t, findSDKKeyStatus(envStatus.SDKKeys, sdks.ObscureKey(string(anchorSDKKey)))) + assert.NotNil(t, findSDKKeyStatus(envStatus.SDKKeys, sdks.ObscureKey(string(extraSDKKey)))) + assert.NotNil(t, findSDKKeyStatus(envStatus.MobileKeys, sdks.ObscureKey(string(anchorMobileKey)))) + assert.NotNil(t, findSDKKeyStatus(envStatus.MobileKeys, sdks.ObscureKey(string(extraMobileKey)))) + + // Exact counts, so a filtered key surfacing under an unexpected obscured value is still caught. + assert.Len(t, envStatus.SDKKeys, 2) + assert.Len(t, envStatus.MobileKeys, 2) +} + +// assertViewScopedKeysRejectedWarning verifies the ingestion WARN names the environment and each +// rejected identifier. The identifier is the non-secret wire name, so it is logged unobscured — an +// operator needs it to find the key in the LaunchDarkly UI. +// +// It also pins that the WARN is emitted exactly once. The design deliberately keeps +// StreamManager.validateCredentialPayload silent — it runs on every environment of every payload — so +// logging from there would double every one of these. A count assertion is what makes that regress +// loudly instead of silently. +func assertViewScopedKeysRejectedWarning(t *testing.T, mockLog *ldlogtest.MockLog) { + t.Helper() + pattern := multiKeyIdentifiers.GetDisplayName() + ".*rejecting credentials scoped to a view: " + + viewScopedSDKID + ", " + viewScopedMobID + mockLog.AssertMessageMatch(t, true, ldlog.Warn, pattern) + + re := regexp.MustCompile(pattern) + matches := 0 + for _, line := range mockLog.GetOutput(ldlog.Warn) { + if re.MatchString(line) { + matches++ + } + } + assert.Equal(t, 1, matches, "the view-scoped WARN must be logged exactly once per payload") +} + +// A view-scoped key never enters the accepted set: it is rejected downstream while the anchor and the +// non-view-scoped sibling in the same environment keep working. + +func TestConcurrentKeysRAC_ViewScopedKeysAreRejected(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent( + multiKeyEnvRep(viewScopedSDKKeyReps(), viewScopedMobileKeyReps(), 1)) + autoConfTest(t, testAutoConfDefaultConfig, &putEvent, func(p autoConfTestParams) { + // The anchor still opens the single upstream client; a rejected sibling changes nothing here. + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) + + env := p.awaitEnvironment(multiKeyEnvID) + + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(false, viewScopedSDKKey, viewScopedMobileKey, "") + + // Absent from the accepted set entirely. That set is the single source for /status, event + // forwarding, and the expiry ticker's schedule, so a filtered key can reach none of them. + accepted := env.GetAcceptedKeys() + assert.NotContains(t, accepted.Server, viewScopedSDKKey) + assert.NotContains(t, accepted.Mobile, viewScopedMobileKey) + + assertViewScopedKeysAbsentFromStatus(t, p.relay) + assertViewScopedKeysRejectedWarning(t, p.mockLog) + }) +} + +func TestConcurrentKeysOffline_ViewScopedKeysAreRejected(t *testing.T) { + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv( + viewScopedAcceptedSDKKeys(), viewScopedAcceptedMobileKeys())) + + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) + + env := p.awaitEnvironment(multiKeyEnvID) + + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + p.assertSDKEndpointsAvailability(false, viewScopedSDKKey, viewScopedMobileKey, "") + + accepted := env.GetAcceptedKeys() + assert.NotContains(t, accepted.Server, viewScopedSDKKey) + assert.NotContains(t, accepted.Mobile, viewScopedMobileKey) + + assertViewScopedKeysAbsentFromStatus(t, p.relay) + assertViewScopedKeysRejectedWarning(t, p.mockLog) + }) +} + +// A malformed payload that also carries view-scoped keys logs the malformed error and stays silent +// about the view-scoped ones. +// +// The handler discards the whole payload and preserves the previous credentials in that case, so it +// never actually rejected anything — a WARN naming keys would describe an action that did not happen. +// BuildAcceptedSet enforces this by returning an empty ViewScopedKeys on every error path; this pins +// that the handler honors it, which moving the log call above the error branch would break. +func TestConcurrentKeysOffline_MalformedPayloadSuppressesViewScopedWarning(t *testing.T) { + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + // The anchor is absent from the accepted SDK keys — structurally malformed — alongside a + // view-scoped entry that would otherwise be reported as rejected. + p.updateHandler.AddEnvironment(multiKeyArchiveEnv( + []envfactory.AcceptedSDKKey{ + {Key: "other-sdk", Value: config.SDKKey("sdk-not-the-anchor")}, + {Key: viewScopedSDKID, Value: viewScopedSDKKey, HasViews: true}, + }, + []envfactory.AcceptedMobileKey{{Key: "anchor-mob", Value: anchorMobileKey}}, + )) + + _ = p.awaitEnvironment(multiKeyEnvID) + + p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "Malformed credential payload for offline environment") + p.mockLog.AssertMessageMatch(t, false, ldlog.Warn, "rejecting credentials scoped to a view") + }) +} + +// A key that gains a view mid-session is revoked on the next payload, and a connected SDK using it is +// disconnected. +// +// This needs no new production code: reconcileAcceptedKeys revokes any key absent from the desired set +// immediately rather than on an expiry timestamp, and RemoveConnectionMapping unmaps before the streams +// are torn down so a reconnect is rejected. These tests assert that behavior rather than build it. The +// live teardown is verified on the offline path, which uses a real SDK client that actually serves +// stream data (mirroring TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyExpires). +func TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyGainsView(t *testing.T) { + // The server-side stream (/all) emits "put"; the mobile streams (/meval, /mping) emit "ping". + run := func(t *testing.T, streamPath, firstEvent string, connectKey credential.SDKCredential, viewOnSDK bool) { + // Named entries, unlike the shared default* fixtures — the WARN interpolates the wire + // identifier, so this is also what lets the log assertion below name a specific key. + const ( + revokedSDKID = "extra-sdk" + revokedMobID = "extra-mob" + ) + named := func(viewScoped bool) ([]envfactory.AcceptedSDKKey, []envfactory.AcceptedMobileKey) { + sdkKeys := []envfactory.AcceptedSDKKey{ + {Key: "anchor-sdk", Value: anchorSDKKey}, + {Key: revokedSDKID, Value: extraSDKKey}, + } + mobileKeys := []envfactory.AcceptedMobileKey{ + {Key: "anchor-mob", Value: anchorMobileKey}, + {Key: revokedMobID, Value: extraMobileKey}, + } + if viewOnSDK { + sdkKeys[1].HasViews = viewScoped + } else { + mobileKeys[1].HasViews = viewScoped + } + return sdkKeys, mobileKeys + } + + offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(named(false))) + _ = p.awaitClient() + env := p.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + req := sharedtest.BuildRequestWithAuth("GET", streamPath, connectKey, nil) + sharedtest.WithStreamRequest(t, req, p.relay, func(eventCh <-chan eventsource.Event) { + // Confirm the stream is live before the key gains a view. + sharedtest.AwaitEventOfType(t, eventCh, firstEvent, 5*time.Second) + + // Reload the archive with the connected non-anchor key now scoped to a view, keeping + // the anchor and primary untouched. + p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv(named(true))) + + // Revocation is immediate on reconcile — no expiry timestamp and no cleanup ticker. + awaitStreamClosed(t, eventCh, 5*time.Second) + }) + + // The revoked key no longer authenticates; the anchor and primary are undisturbed. + revokedID := revokedSDKID + if viewOnSDK { + p.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") + } else { + revokedID = revokedMobID + p.assertSDKEndpointsAvailability(false, "", extraMobileKey, "") + } + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + + // The offline update handler logs the rejection too, not just the add handler. + p.mockLog.AssertMessageMatch(t, true, ldlog.Warn, + multiKeyIdentifiers.GetDisplayName()+".*rejecting credentials scoped to a view: "+revokedID) + + // Losing the view re-admits the key: the filter is stateless, not a one-way latch. This is + // the remediation an operator performs after seeing the WARN, so it has to work. + p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv(named(false))) + if viewOnSDK { + p.assertSDKEndpointsAvailability(true, extraSDKKey, "", "") + } else { + p.assertSDKEndpointsAvailability(true, "", extraMobileKey, "") + } + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + }) + } + + t.Run("sdk key", func(t *testing.T) { run(t, "/all", "put", extraSDKKey, true) }) + t.Run("mobile key", func(t *testing.T) { + // base64 of {"key":"userkey","kind":"user"} — a valid context, not the legacy user format. + run(t, "/meval/eyJrZXkiOiJ1c2Vya2V5Iiwia2luZCI6InVzZXIifQ==", "ping", extraMobileKey, false) + }) +} + +// The RAC equivalent, with an open downstream stream held on the anchor while the non-anchor key gains +// a view around it: the revoked key stops routing, a reconnect with it is rejected, and the anchor's +// live connection is untouched. Mirrors TestConcurrentKeysRAC_ArrayPatchAddsAndRemovesNonAnchorKeys. +func TestConcurrentKeysRAC_KeyGainingViewIsRevokedAndOthersUndisturbed(t *testing.T) { + putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) + racMock := configsource.NewRACMock(t, &putEvent) + + cfg := config.Config{AutoConfig: config.AutoConfigConfig{Key: testAutoConfKey}} + cfg.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(racMock.URL) + // A short cleanup interval so the expiry ticker runs during the test: a revoked view-scoped key + // must be gone outright, never left scheduled for a later drop. + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + relay, err := newRelayInternal(cfg, relayInternalOptions{ + loggers: mockLog.Loggers, + clientFactory: testclient.CreateDummyClient, + }) + require.NoError(t, err) + defer relay.Close() + + h := relayTestHelper{t: t, relay: relay} + env := h.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + h.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") + + req := sharedtest.BuildRequestWithAuth("GET", "/all", anchorSDKKey, nil) + sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) + + // One patch marks both non-anchor entries as view-scoped, keeping the anchor and primary. + patch := multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-sdk", Value: string(anchorSDKKey)}, + {Key: "extra-sdk", Value: string(extraSDKKey), HasViews: true}, + }, + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-mob", Value: string(anchorMobileKey)}, + {Key: "extra-mob", Value: string(extraMobileKey), HasViews: true}, + }, + 2, + ) + racMock.Send(configsource.MakeAutoConfigPatchEvent(patch)) + + // Both stop routing, so a reconnect presenting either one is rejected. + require.Eventually(t, func() bool { + _, errSDK := relay.getEnvironment(sdkauth.New(extraSDKKey)) + _, errMobile := relay.getEnvironment(sdkauth.New(extraMobileKey)) + return errSDK != nil && errMobile != nil + }, 5*time.Second, 5*time.Millisecond, "keys that gained a view were not revoked") + + // The anchor's open stream is undisturbed by the revocation alongside it. + assertStreamStaysOpen(t, eventCh, 500*time.Millisecond) + }) + + // The anchor never changed, so no re-anchor happened and it still owns the connection. + assert.Equal(t, anchorSDKKey, env.GetAcceptedKeys().Anchor) + h.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + h.assertSDKEndpointsAvailability(false, extraSDKKey, extraMobileKey, "") + + mockLog.AssertMessageMatch(t, true, ldlog.Warn, + multiKeyIdentifiers.GetDisplayName()+".*rejecting credentials scoped to a view: extra-sdk, extra-mob") +} From 2781baa7d74c0dd6e6fdfa8f3c60832fbe42d948 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Wed, 12 Aug 2026 11:34:07 -0700 Subject: [PATCH 57/66] chore(release): set version to 8.21.0-rc.1 --- relay/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/version/version.go b/relay/version/version.go index 0248e89a..32c3d1b8 100644 --- a/relay/version/version.go +++ b/relay/version/version.go @@ -2,4 +2,4 @@ package version // Version is the package version -const Version = "8.20.0-rc.1" // {{ x-release-please-version }} +const Version = "8.21.0-rc.1" // {{ x-release-please-version }} From 718b55d546d759734b7327dc07db1a2743bfd0dd Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Fri, 14 Aug 2026 08:46:04 -0700 Subject: [PATCH 58/66] chore(release): align version.go with v8 ahead of the merge v8 has since shipped 8.20.0, leaving version.go as the only conflicting file between this branch and v8. Setting it back to v8's value makes both sides of the merge an identical change, so the merge applies cleanly. The 8.21.0-rc.1 string is preserved by the tag of the same name, and release-please will bump this file when it cuts the GA release. --- relay/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/version/version.go b/relay/version/version.go index 32c3d1b8..050cb499 100644 --- a/relay/version/version.go +++ b/relay/version/version.go @@ -2,4 +2,4 @@ package version // Version is the package version -const Version = "8.21.0-rc.1" // {{ x-release-please-version }} +const Version = "8.20.0" // {{ x-release-please-version }} From 31ddfbab49fb2eaf1d7f209e47c1b2dbf646110b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:37:55 -0400 Subject: [PATCH 59/66] fix(deps): bump supported Go versions to 1.26.6 and 1.25.13 (#820) (cherry picked from commit 497509878a1809a15e61cecb1729facdbafb68e9) --- .github/variables/go-versions.env | 4 ++-- Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/variables/go-versions.env b/.github/variables/go-versions.env index a90cd467..fa14bcd8 100644 --- a/.github/variables/go-versions.env +++ b/.github/variables/go-versions.env @@ -1,2 +1,2 @@ -latest=1.26.5 -penultimate=1.25.12 +latest=1.26.6 +penultimate=1.25.13 diff --git a/Dockerfile b/Dockerfile index 29b9edf1..e4fc1614 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # This is a standalone Dockerfile that does not depend on goreleaser building the binary # It is NOT the version that is pushed to dockerhub -FROM golang:1.26.5-alpine3.24 as builder +FROM golang:1.26.6-alpine3.24 as builder # See "Runtime platform versions" in CONTRIBUTING.md RUN apk --no-cache add \ From 9d35f816ddceb09c39f0a08b37b4450e3a66194c Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Mon, 17 Aug 2026 12:33:37 -0400 Subject: [PATCH 60/66] docs: Simplify comments per ASD-STE100 (#825) --- config/config.go | 14 +- config/config_validation.go | 3 +- internal/api/status_reps.go | 22 +- internal/autoconfig/stream_manager.go | 33 +- internal/credential/accepted_set.go | 81 ++--- internal/credential/accepted_set_builder.go | 18 +- internal/credential/rotator.go | 190 ++++------- internal/envfactory/env_params.go | 14 +- internal/envfactory/env_rep.go | 61 +--- internal/envfactory/reconcile_helper.go | 72 ++--- internal/events/event_publisher.go | 16 +- internal/relayenv/env_context.go | 30 +- internal/relayenv/env_context_impl.go | 318 ++++++------------- internal/sdks/client_factory.go | 5 +- internal/sharedtest/configsource/rac_mock.go | 2 +- internal/store/relay_feature_store.go | 45 ++- relay/autoconfig_actions.go | 6 +- relay/endpoints_status.go | 20 +- relay/filedata_actions.go | 6 +- 19 files changed, 326 insertions(+), 630 deletions(-) diff --git a/config/config.go b/config/config.go index 5fe62830..d892f78c 100644 --- a/config/config.go +++ b/config/config.go @@ -38,15 +38,13 @@ const ( // DefaultEventCapacity is the default value for EventsConfig.Capacity if not specified. DefaultEventCapacity = 1000 - // DefaultMetricsCapacity is the default value for EventsConfig.MetricsCapacity if not specified. - // This is the maximum queue capacity for the usage-metrics event publisher, which emits one event - // per concurrent unique connection on each flush. It is set well above DefaultEventCapacity because - // high-concurrency nodes routinely exceed 1000 unique connections. + // DefaultMetricsCapacity is the default for EventsConfig.MetricsCapacity: the maximum queue size + // for the usage-metrics publisher, which emits one event per unique connection on each flush. + // It exceeds DefaultEventCapacity because high-concurrency nodes exceed 1000 connections. DefaultMetricsCapacity = 10000 // DefaultMetricsInitialCapacity is the number of events the usage-metrics publisher queue - // preallocates space for. The queue grows on demand from this size up to MetricsCapacity, so that - // the higher maximum does not reserve all of its memory up front on nodes that never reach it. + // preallocates space for. The queue grows on demand from this size up to MetricsCapacity. DefaultMetricsInitialCapacity = 1000 // DefaultHeartbeatInterval is the default value for MainConfig.HeartBeatInterval if not specified. @@ -107,9 +105,7 @@ const ( // It likely doesn't make sense to use an interval this frequent in production use-cases. minimumCredentialCleanupInterval = 100 * time.Millisecond // minimumMetricsCapacity is the smallest value accepted for EventsConfig.MetricsCapacity. Usage - // metrics are how LaunchDarkly reports on account usage, so we do not allow the maximum queue - // capacity to be shrunk below the historical default of 1000; smaller configured values are - // clamped up to this floor. + // metrics report account usage, so the floor is the historical default of 1000. minimumMetricsCapacity = 1000 ) diff --git a/config/config_validation.go b/config/config_validation.go index 6fd17a56..be0226d9 100644 --- a/config/config_validation.go +++ b/config/config_validation.go @@ -249,8 +249,7 @@ func validateMaxClientRequestBodySize(result *ct.ValidationResult, c *Config) { } // validateMetricsCapacity enforces the minimum queue capacity for the usage-metrics event publisher. -// Rather than fail startup on a too-small value, it clamps the value up to the minimum and warns, so -// that a misconfiguration never prevents Relay from running while still protecting usage telemetry. +// Rather than fail startup on a too-small value, it clamps the value up to the minimum and warns. func validateMetricsCapacity(c *Config, loggers ldlog.Loggers) { if !c.Events.MetricsCapacity.IsDefined() { return diff --git a/internal/api/status_reps.go b/internal/api/status_reps.go index 6d72298f..6e104346 100644 --- a/internal/api/status_reps.go +++ b/internal/api/status_reps.go @@ -16,13 +16,11 @@ type StatusRep struct { } // KeyStatus is the JSON representation of one accepted SDK or mobile key in the status endpoint's -// sdkKeys[] / mobileKeys[] arrays. +// sdkKeys[] and mobileKeys[] arrays. // -// Key is the non-secret human-readable identifier from the wire format (the "key" field of a -// sdkKeys/mobileKeys entry); it is omitted when the source carried no identifier (manual config, or -// an old-format payload predating concurrent keys). Value is the obscured credential secret (via -// sdks.ObscureKey). Expiry carries the Unix-millisecond expiry timestamp when the key is being phased -// out; it is omitted for permanent keys. +// Key is the non-secret wire identifier, omitted when the source carried none. Value is the credential +// secret, obscured by sdks.ObscureKey. Expiry is the expiry timestamp in Unix milliseconds, omitted +// for permanent keys. type KeyStatus struct { Key string `json:"key,omitempty"` Value string `json:"value"` @@ -33,12 +31,10 @@ type KeyStatus struct { // // This is exported for use in integration test code. type EnvironmentStatusRep struct { - // SDKKey is the obscured anchor SDK key — the key relay uses for its upstream connection. It - // designates which SDKKeys entry is the anchor. + // SDKKey is the obscured anchor SDK key. It designates which SDKKeys entry is the anchor. SDKKey string `json:"sdkKey"` - // SDKKeys carries the full accepted set of server-side SDK keys — including the anchor — with their - // identifiers, obscured values, and optional expiry. Always present; always contains at least the - // anchor. + // SDKKeys carries the full accepted set of server-side SDK keys, including the anchor. It is always + // present and always contains at least the anchor. SDKKeys []KeyStatus `json:"sdkKeys"` EnvID string `json:"envId,omitempty"` EnvKey string `json:"envKey,omitempty"` @@ -47,8 +43,8 @@ type EnvironmentStatusRep struct { ProjName string `json:"projName,omitempty"` // MobileKey is the obscured primary mobile key. It designates which MobileKeys entry is the primary. MobileKey string `json:"mobileKey,omitempty"` - // MobileKeys carries the full accepted set of mobile keys — including the primary. Always present; - // empty for an environment with no mobile keys (e.g. server-side only). + // MobileKeys carries the full accepted set of mobile keys, including the primary. It is always + // present, and empty for an environment with no mobile keys. MobileKeys []KeyStatus `json:"mobileKeys"` ExpiringSDKKey string `json:"expiringSdkKey,omitempty"` Status string `json:"status"` diff --git a/internal/autoconfig/stream_manager.go b/internal/autoconfig/stream_manager.go index 5c15b845..849131f5 100644 --- a/internal/autoconfig/stream_manager.go +++ b/internal/autoconfig/stream_manager.go @@ -381,7 +381,7 @@ func (s *StreamManager) handleStreamEvent(event es.Event) bool { s.loggers.Infof(logMsgWrongPath, PutEvent, putMessage.Path) break } - // The stream has authoritative data — cancel any in-flight cache read. + // The stream has authoritative data, so cancel any in-flight cache read. if s.cacheCancel != nil { s.cacheCancel() s.cacheCancel = nil @@ -481,7 +481,6 @@ func (s *StreamManager) handleStreamEvent(event es.Event) bool { return shouldRestart } -// dispatchEnvAction dispatches a single environment action to the handler. func (s *StreamManager) dispatchEnvAction(id config.EnvironmentID, rep envfactory.EnvironmentRep, action Action) { switch action { case ActionNoop: @@ -498,9 +497,9 @@ func (s *StreamManager) dispatchEnvAction(id config.EnvironmentID, rep envfactor } // validateCredentialPayload checks that an environment rep carries a structurally valid credential -// set. It runs at the stream parse boundary, before the rep's version is recorded via Upsert: a -// malformed payload must not advance the version, or the backend's fresh put — which carries the same -// version — would be deduplicated away by the MessageReceiver. +// set. It runs at the stream parse boundary, before Upsert records the rep's version. A malformed +// payload must not advance the version: the backend's fresh put carries the same version, and the +// MessageReceiver would deduplicate it away. func (s *StreamManager) validateCredentialPayload(rep envfactory.EnvironmentRep) error { _, _, err := envfactory.BuildAcceptedSet(rep.ToParams()) return err @@ -529,8 +528,8 @@ func (s *StreamManager) applyCachedContent(content *PutContent) { // All of the private methods below can be assumed to be called from the same goroutine that consumeStream // is on. We will never be processing more than one stream message at the same time. // -// handlePut returns true if the stream should be restarted — a malformed credential payload in any of -// the environments triggers a reconnect, while still processing the well-formed ones. +// handlePut returns true if the stream should be restarted. A malformed credential payload in any +// environment triggers a reconnect, and the well-formed environments are still processed. func (s *StreamManager) handlePut(content PutContent) bool { // A "put" message represents a full environment set. We will compare them one at a time to the // current set of environments (if any), calling the handler's AddEnvironment for any new ones, @@ -544,8 +543,7 @@ func (s *StreamManager) handlePut(content PutContent) bool { s.loggers.Warnf(logMsgEnvHasWrongID, rep.EnvID, id) continue } - // Validate before Upsert so a malformed payload does not advance the version (see - // validateCredentialPayload). Skip this env, preserving its previous state, and reconnect. + // See handleStreamEvent: validate before Upsert. Skip this env and reconnect. if err := s.validateCredentialPayload(rep); err != nil { s.loggers.Errorf("Received malformed credential payload for environment %q (%s); preserving previous credentials and will restart stream", rep.EnvID, err) shouldRestart = true @@ -582,19 +580,14 @@ func (s *StreamManager) handlePut(content PutContent) bool { return shouldRestart } -// persistPut writes a put's content to the cache. A clean put is stored atomically with SetAll. When -// the put carried malformed environments, a plain SetAll would corrupt the cache: filtering the -// malformed envs out would drop them, and if every env was malformed it would wipe the cache entirely. -// Instead we keep each malformed env's previously-cached entry and write the resulting snapshot — so -// the valid envs and the filters in this put are persisted, the malformed envs keep their last-good -// value (or are simply omitted when the cache has no prior entry for them), and envs the put removed -// are still dropped. The reconnect a malformed put triggers fetches fresh data for the malformed envs. -// If the prior cache genuinely can't be read, leave it untouched rather than risk dropping entries. +// persistPut writes a put's content to the cache with SetAll. When the put carried malformed +// environments, persistPut first substitutes each malformed env's previously-cached entry, because a +// plain SetAll would drop those envs, or wipe the cache if every env was malformed. Envs the put +// removed are still dropped. If the prior cache cannot be read, persistPut leaves it untouched. func (s *StreamManager) persistPut(content PutContent, malformedEnvIDs map[config.EnvironmentID]bool) { if len(malformedEnvIDs) > 0 { - // A nil result with no error means an empty cache (per the Cache contract), not a failure — - // in that case there are simply no prior entries to restore. Only a real read error makes it - // unsafe to rewrite the snapshot, so bail out only then. + // A nil result with no error means an empty cache, not a failure, so there are no prior entries + // to restore. Only a read error makes it unsafe to rewrite the snapshot. prev, err := s.cache.GetAll(context.Background()) if err != nil { s.loggers.Warnf("Skipping AutoConfig cache write for a put with malformed credentials (cannot read prior cache): %v", err) diff --git a/internal/credential/accepted_set.go b/internal/credential/accepted_set.go index c3011929..a7fe5188 100644 --- a/internal/credential/accepted_set.go +++ b/internal/credential/accepted_set.go @@ -6,28 +6,19 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" ) -// AcceptedSet is the full set of credentials that an environment should accept after a reconcile. -// It carries every accepted server-side SDK key and mobile key — each with an optional per-key -// expiry — plus the single environment ID and two primary designations: +// AcceptedSet is the full set of credentials an environment accepts after a reconcile: every +// server-side SDK key and mobile key with an optional expiry, the environment ID, and two +// designations. // -// - The anchor: the one SDK key that owns the environment's upstream connection. Set with -// WithAnchor. -// - The primary mobile key: the singular default mobile key (the wire's mobKey), used where one -// mobile key is required, e.g. event forwarding. Set with WithPrimaryMobileKey. +// - The anchor is the one SDK key that owns the environment's upstream connection. +// - The primary mobile key is the default used where one mobile key is required, such as event +// forwarding. // -// WithAnchor / WithPrimaryMobileKey both add the key to the set and designate it, so adding a -// single key takes one call. Build requires that an anchor was designated. (Structural validation of -// the wire payload — undefined credentials, a designated key absent from its array — happens upstream -// when the payload is parsed into the set.) -// -// A key's expiry is taken from its entry in this set; the legacy sdkKey.expiring{} wire slot is not -// consulted when building it. -// -// Construct an AcceptedSet with AcceptedSetBuilder (see accepted_set_builder.go). +// Construct an AcceptedSet with AcceptedSetBuilder. Structural validation of the wire payload happens +// upstream, in BuildAcceptedSet. type AcceptedSet struct { - // sdkKeys and mobileKeys store each accepted key once, keyed by value (the secret), so duplicates - // collapse without a containment scan. The map value carries the key's metadata (see - // AcceptedKey). A nil map is a valid empty set (reads return absent; only the builder writes). + // sdkKeys and mobileKeys store each accepted key once, keyed by value, so duplicates collapse. A + // nil map is a valid empty set, and only the builder writes. sdkKeys map[config.SDKKey]AcceptedKey anchor config.SDKKey mobileKeys map[config.MobileKey]AcceptedKey @@ -48,27 +39,11 @@ func (s AcceptedSet) hasMobileKey(key config.MobileKey) bool { } // MalformedCredentialSetError is returned when a credential payload cannot produce a valid -// AcceptedSet. This covers: -// -// 1. The anchor SDK key (sdkKey.value) is absent or undefined, or defined but not present in -// sdkKeys[] — a violation of the invariant that the designated anchor is one of the accepted keys. -// 2. The primary mobile key (mobKey) is defined but not present in mobileKeys[] — the mobile-key -// analogue of the anchor invariant. -// 3. mobileKeys[] is non-empty but no primary mobile key (mobKey) is designated — accepting it would -// clear the environment's primary mobile key on reconcile with no repoint, so event forwarding -// would keep using the previous (possibly revoked) primary. (No mobile keys at all is valid.) -// 4. An entry in sdkKeys[] or mobileKeys[] has an empty value — a credential that would be -// accepted by relay but can never authenticate any SDK. -// 5. No SDK key survived at all, so the environment would have nothing to authenticate with. A -// payload reaches this by combining an undefined anchor with an sdkKeys[] array that is either -// empty or entirely made up of keys relay excludes, such as keys scoped to a view. +// AcceptedSet. Each constructor below documents one cause. // -// Validation happens before Reconcile is called; Rotator.Reconcile trusts the set it is handed. -// Because the error is raised before any state mutation, the environment's previous accepted set is -// preserved automatically. The caller is responsible for the second half of the malformed-payload -// policy: reconnecting the RAC stream with jitter to force a fresh put. RAC is one-way push with no -// NAK channel, so without the reconnect the backend would believe the malformed patch was applied -// and would not send fresh state. +// Validation runs before Reconcile, so the environment keeps its previous accepted set. The caller +// must also reconnect the RAC stream with jitter: RAC is one-way push with no NAK channel, so +// without a reconnect the backend assumes the patch was applied and sends nothing new. type MalformedCredentialSetError struct { // msg is the human-readable description set by the constructor. msg string @@ -83,40 +58,32 @@ func newMissingAnchorError() *MalformedCredentialSetError { return &MalformedCredentialSetError{msg: "malformed credential set: anchor SDK key is missing"} } -// newNoSDKKeysError returns a MalformedCredentialSetError for a set that ended up with no SDK key at -// all. The message describes the payload rather than the builder, because that is what an operator -// reading the log can act on. +// newNoSDKKeysError returns a MalformedCredentialSetError for a set with no usable SDK key. func newNoSDKKeysError() *MalformedCredentialSetError { return &MalformedCredentialSetError{msg: "malformed credential set: no usable SDK key in sdkKeys[]"} } -// NewAnchorNotInSetError returns a MalformedCredentialSetError for a payload whose designated anchor -// (sdkKey.value) is defined but not present in the sdkKeys[] array — a structural inconsistency. The -// anchor value is a secret, so it is deliberately not included in the message. +// NewAnchorNotInSetError reports a payload whose designated anchor (sdkKey.value) is defined but +// absent from sdkKeys[]. Credential values are secrets, so no message here includes one. func NewAnchorNotInSetError() *MalformedCredentialSetError { return &MalformedCredentialSetError{msg: "malformed credential set: anchor SDK key is not present in sdkKeys[]"} } -// NewPrimaryMobileKeyNotInSetError returns a MalformedCredentialSetError for a payload whose -// designated primary mobile key (mobKey) is defined but not present in the mobileKeys[] array — the -// mobile-key analogue of NewAnchorNotInSetError. The key value is a secret, so it is deliberately not -// included in the message. +// NewPrimaryMobileKeyNotInSetError reports a payload whose designated primary mobile key (mobKey) is +// defined but absent from mobileKeys[]. func NewPrimaryMobileKeyNotInSetError() *MalformedCredentialSetError { return &MalformedCredentialSetError{msg: "malformed credential set: primary mobile key is not present in mobileKeys[]"} } -// NewPrimaryMobileKeyMissingError returns a MalformedCredentialSetError for a payload that carries a -// non-empty mobileKeys[] array but leaves the primary mobile key (mobKey) undefined — no default is -// designated. Accepting it would clear the environment's primary mobile key on reconcile with no -// repoint, so event forwarding would keep using the previous (possibly revoked) primary. There are no -// secrets to omit from the message. +// NewPrimaryMobileKeyMissingError reports a payload with a non-empty mobileKeys[] and no designated +// primary. Accepting it would clear the primary without a repoint, leaving event forwarding on the +// previous key. func NewPrimaryMobileKeyMissingError() *MalformedCredentialSetError { return &MalformedCredentialSetError{msg: "malformed credential set: mobileKeys[] is non-empty but no primary mobile key is designated"} } -// NewEmptyCredentialError returns a MalformedCredentialSetError for a key-array entry whose -// value field is empty. kind is "sdkKeys" or "mobileKeys"; key is the entry's wire "key" identifier -// (may be empty for old-format payloads that synthesize from the singular fields). +// NewEmptyCredentialError reports a key-array entry whose value field is empty. kind is "sdkKeys" or +// "mobileKeys"; key is the entry's wire identifier, which old-format payloads leave empty. func NewEmptyCredentialError(kind, key string) *MalformedCredentialSetError { if key == "" { return &MalformedCredentialSetError{ diff --git a/internal/credential/accepted_set_builder.go b/internal/credential/accepted_set_builder.go index d6332749..52c7f73f 100644 --- a/internal/credential/accepted_set_builder.go +++ b/internal/credential/accepted_set_builder.go @@ -47,10 +47,9 @@ func (b *AcceptedSetBuilder) WithSDKKey(p SDKKeyParams) *AcceptedSetBuilder { return b } -// WithAnchor adds p.Value and designates it as the anchor — the SDK key that owns the environment's -// upstream connection. The anchor is always permanent, so p.Expiry is ignored. It is a no-op if the -// value is undefined. Unlike WithSDKKey it overwrites any existing entry for the value, since -// designating the anchor takes precedence over an earlier non-anchor add. +// WithAnchor adds p.Value and designates it as the anchor. The anchor is always permanent, so +// p.Expiry is ignored. It is a no-op if the value is undefined. Unlike WithSDKKey, it overwrites an +// existing entry for the value. func (b *AcceptedSetBuilder) WithAnchor(p SDKKeyParams) *AcceptedSetBuilder { if !p.Value.Defined() { return b @@ -69,9 +68,8 @@ func (b *AcceptedSetBuilder) WithMobileKey(p MobileKeyParams) *AcceptedSetBuilde return b } -// WithPrimaryMobileKey adds p.Value and designates it as the primary mobile key — the singular -// default (the wire's mobKey) used where one mobile key is required, e.g. event forwarding. The -// primary is always permanent, so p.Expiry is ignored. It is a no-op if the value is undefined. +// WithPrimaryMobileKey adds p.Value and designates it as the primary mobile key. The primary is +// always permanent, so p.Expiry is ignored. It is a no-op if the value is undefined. func (b *AcceptedSetBuilder) WithPrimaryMobileKey(p MobileKeyParams) *AcceptedSetBuilder { if !p.Value.Defined() { return b @@ -89,10 +87,8 @@ func (b *AcceptedSetBuilder) WithEnvironmentID(id config.EnvironmentID) *Accepte return b } -// Build validates and returns the accumulated AcceptedSet. It returns a -// *MalformedCredentialSetError if no SDK key was added, or if no anchor was designated (via -// WithAnchor). Because WithAnchor also adds the key, a designated anchor is always among the -// accepted SDK keys. +// Build validates and returns the accumulated AcceptedSet. It returns a *MalformedCredentialSetError +// if no SDK key was added, or if no anchor was designated. func (b *AcceptedSetBuilder) Build() (AcceptedSet, error) { if len(b.set.sdkKeys) == 0 { return AcceptedSet{}, newNoSDKKeysError() diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index b9bab760..544a0e36 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -10,24 +10,19 @@ import ( "github.com/launchdarkly/ld-relay/v8/config" ) -// AcceptedKey is the metadata for one accepted credential: its optional expiry and optional wire -// "key" identifier. The credential value itself is the map key wherever AcceptedKey is stored — the -// rotator's accepted-key maps, the builder's AcceptedSet, and the AcceptedKeySet returned by -// AcceptedKeys. +// AcceptedKey is the metadata for one accepted credential: its optional expiry and its optional wire +// identifier. The credential value itself is the map key wherever AcceptedKey is stored. type AcceptedKey struct { // Expiry is the key's expiry. A nil expiry means the key is permanent. Expiry *time.Time - // Key is the non-secret wire "key" identifier — a human-readable name. Nil when the source carried - // none (manual configuration, or an old-format payload predating concurrent keys). + // Key is the non-secret wire identifier, a human-readable name. Nil when the source carried none + // (manual configuration, or an old-format payload predating concurrent keys). Key *string } -// AcceptedKeySet is a point-in-time snapshot of an environment's full accepted credential set, -// returned by Rotator.AcceptedKeys. Server and Mobile are keyed by credential value (the secret); -// the value AcceptedKey carries that key's metadata. Anchor and PrimaryMobile name the designated -// keys within Server and Mobile. The status endpoint maps Server/Mobile to the sdkKeys[]/mobileKeys[] -// arrays and uses Anchor to mark the anchor entry. Reads of the maps and the designations are taken -// under a single lock, so they are mutually consistent. +// AcceptedKeySet is a point-in-time snapshot of an environment's full accepted set, returned by +// Rotator.AcceptedKeys. Server and Mobile are keyed by credential value; Anchor and PrimaryMobile +// name the designated keys within them. Reads are taken under one lock, so the fields agree. type AcceptedKeySet struct { Server map[config.SDKKey]AcceptedKey Mobile map[config.MobileKey]AcceptedKey @@ -53,7 +48,6 @@ type Rotator struct { acceptedSDKKeys map[config.SDKKey]AcceptedKey // acceptedMobileKeys is the full set of accepted mobile keys with optional per-key expiry. - // A nil expiry means the key is permanent. acceptedMobileKeys map[config.MobileKey]AcceptedKey expirations []SDKCredential @@ -103,7 +97,7 @@ func (r *Rotator) MobileKey() config.MobileKey { return r.primaryMobileKey } -// AnchorKey returns the anchor SDK key — the key that owns the upstream connection. +// AnchorKey returns the anchor SDK key, which owns the upstream connection. func (r *Rotator) AnchorKey() config.SDKKey { r.mu.RLock() defer r.mu.RUnlock() @@ -133,12 +127,8 @@ func (r *Rotator) allCredentials() []SDKCredential { return creds } -// DeprecatedCredentials returns the SDK keys being phased out — every accepted SDK key, other than the -// anchor, that carries a future expiry. (Per-key expiry is stored as data on the accepted entry; the -// cleanup ticker drops the key once it elapses.) -// -// Mobile keys are deliberately not returned even though they expire the same way SDK keys do — carried -// as per-key expiry and dropped by the same cleanup ticker. +// DeprecatedCredentials returns every accepted SDK key, other than the anchor, that carries an +// expiry. It does not return mobile keys, which expire the same way. func (r *Rotator) DeprecatedCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() @@ -152,9 +142,7 @@ func (r *Rotator) DeprecatedCredentials() []SDKCredential { return out } -// AllCredentials returns every accepted credential: every accepted SDK key, every accepted mobile -// key (including those carrying a future expiry — they still authenticate until the cleanup ticker -// drops them), and the environment ID. +// AllCredentials returns every accepted credential: SDK keys, mobile keys, and the environment ID. func (r *Rotator) AllCredentials() []SDKCredential { r.mu.RLock() defer r.mu.RUnlock() @@ -167,22 +155,17 @@ func (r *Rotator) expireSDKKey(sdkKey config.SDKKey) { r.expirations = append(r.expirations, sdkKey) } -// expireMobileKey drops a mobile key from the accepted set and queues its expiration. -// Deleting from acceptedMobileKeys is load-bearing: AllCredentials derives from that map, -// so an expired key would otherwise linger as an accepted credential. Mirrors expireSDKKey. +// expireMobileKey drops a mobile key from the accepted set and queues its expiration. Deleting from +// acceptedMobileKeys is load-bearing: AllCredentials derives from that map. func (r *Rotator) expireMobileKey(mobileKey config.MobileKey) { r.loggers.Infof("Deprecated mobile key %s has expired and is no longer valid for authentication", mobileKey.Masked()) delete(r.acceptedMobileKeys, mobileKey) r.expirations = append(r.expirations, mobileKey) } -// StepTime provides the current time to the Rotator, allowing it to compute the set of additions and -// expirations for the tracked credentials since the last time this method was called. -// -// It enforces per-key expiry for both SDK and mobile keys: expiry is stored as data on the accepted -// entry (AcceptedKey.Expiry); a nil expiry means the key is permanent and is never expired here. -// -// Expiry happens strictly after a key's expiry timestamp. +// StepTime provides the current time to the Rotator, so it can compute the additions and expirations +// for the tracked credentials since the last call. It enforces per-key expiry for SDK and mobile keys, +// strictly after a key's expiry timestamp. func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expirations []SDKCredential) { r.mu.Lock() defer r.mu.Unlock() @@ -209,21 +192,15 @@ func (r *Rotator) StepTime(now time.Time) (additions []SDKCredential, expiration return additions, expirations } -// ReconcileResult signals state changes that the caller must apply synchronously rather than rely -// on the normal addCredential / removeCredential flow driven by StepTime. +// ReconcileResult signals state changes the caller must apply itself, rather than through the +// StepTime-driven addCredential and removeCredential flow. // -// AnchorChange is non-nil when the SDK anchor changed during Reconcile. The rotator does NOT flip -// its anchor pointer in that case — the caller must drive the synchronous re-anchor sequence -// (build the new anchor's SDK client if one does not exist, wait for Initialized, then invoke -// CommitAnchor to atomically move the pointer, then call ReplaceCredential on the event dispatcher -// and metrics publisher, then re-wire big-segment sync). +// AnchorChange is non-nil when the SDK anchor changed. The caller drives the re-anchor sequence; see +// envContextImpl.reanchor. // -// MobilePrimaryRepoint is non-nil when the primary mobile key changed AND the new primary was -// already in the accepted set. In that case it does not appear in StepTime's additions list and -// addCredential's primary-mobile gate will not fire for it, so the caller must invoke -// eventDispatcher.ReplaceCredential synchronously. When nil, either the primary mobile key did not -// change, or it changed to a newly-accepted key — in which case the normal addCredential path -// handles the ReplaceCredential call via the existing gate. +// MobilePrimaryRepoint is non-nil when the primary mobile key changed to a key already accepted. Such +// a key is not in additions, so addCredential does not repoint the event dispatcher for it and the +// caller must do so. type ReconcileResult struct { AnchorChange *AnchorChange MobilePrimaryRepoint *config.MobileKey @@ -231,37 +208,25 @@ type ReconcileResult struct { // AnchorChange describes an SDK anchor transition produced by Reconcile. // -// NewAnchorPreviouslyAccepted distinguishes the two re-anchor paths: -// - false (the anchor is a new key): the new anchor was not previously in the accepted set. The -// synchronous re-anchor must register the credential mappings (envStreams, handlers, connection -// mapping), construct and initialize a new SDK client, then invoke CommitAnchor + ReplaceCredential. -// - true (the anchor is a previously-accepted key): the new anchor was already accepted (typically -// a former anchor still in its grace period). Its credential mappings are already registered and a -// client may already exist; the synchronous re-anchor reuses it (or constructs one only if missing -// — see the re-anchor sequence in env_context_impl.go), then invokes CommitAnchor + ReplaceCredential. +// NewAnchorPreviouslyAccepted is false when the new anchor is a brand-new key, so the re-anchor must +// register that key's credential mappings. It is true when the key was already accepted, so the +// mappings already exist and a client may too. See envContextImpl.reanchor. type AnchorChange struct { PreviousAnchor config.SDKKey NewAnchor config.SDKKey NewAnchorPreviouslyAccepted bool } -// Reconcile updates the rotator to match set. The set names its own anchor (the primary SDK key) and -// primary mobile key. It diffs the desired accepted set against the current one and queues additions -// and expirations (drained by the next StepTime call); keys newly present are accepted, and keys no -// longer present are revoked. Per-key expiry is stored as data on the accepted entry; the cleanup -// ticker (StepTime) is what later acts on it, dropping a key once its expiry passes. An undefined -// environment ID leaves the current one unchanged, since environments are removed via teardown -// rather than reconcile. +// Reconcile updates the rotator to match set. It queues additions and expirations, which the next +// StepTime call drains. Keys newly present are accepted; keys no longer present are revoked. An +// undefined environment ID leaves the current one unchanged. Per-key expiry is stored on the accepted +// entry, and StepTime acts on it. // -// The set is assumed well-formed: AcceptedSetBuilder.Build validates that an anchor was designated -// (and, because WithAnchor adds the key as it designates it, that the anchor is among the SDK -// keys), so Reconcile trusts what it is handed rather than re-validating. +// set is assumed well-formed: AcceptedSetBuilder.Build already guaranteed a designated anchor. // -// Reconcile does NOT flip the SDK anchor pointer when the anchor changes — the returned -// ReconcileResult.AnchorChange signals the change so the caller can drive the synchronous re-anchor -// sequence, then call CommitAnchor to atomically move the pointer. When the new anchor is a new key -// (NewAnchorPreviouslyAccepted == false) it is also stripped from additions so that the async -// startSDKClient invocation in addCredential does not race the synchronous client build. +// Reconcile does NOT flip the anchor pointer. The returned AnchorChange signals the change; the +// caller drives the re-anchor and then calls CommitAnchor. A brand-new anchor is also stripped from +// additions, because reanchor registers that key's mappings itself. func (r *Rotator) Reconcile(set AcceptedSet, now time.Time) ReconcileResult { r.mu.Lock() defer r.mu.Unlock() @@ -282,12 +247,8 @@ func (r *Rotator) Reconcile(set AcceptedSet, now time.Time) ReconcileResult { r.reconcileSDKKeys(set, now) if result.AnchorChange != nil && !result.AnchorChange.NewAnchorPreviouslyAccepted { - // The anchor is a new key: reconcileAcceptedKeys just appended it to r.additions. Strip it — - // the synchronous re-anchor sequence in env_context_impl owns the new anchor's setup - // (credential mappings + client build + flip + ReplaceCredential). If addCredential drained this - // addition normally, its async startSDKClient would race the synchronous build. When the anchor - // is a previously-accepted key it was already in acceptedSDKKeys, so reconcileAcceptedKeys did - // not add it — no strip needed. + // The anchor is a new key, so reconcileAcceptedKeys appended it to r.additions. Strip it: + // reanchor registers this key's mappings, so draining the addition would register them twice. r.additions = slices.DeleteFunc(r.additions, func(c SDKCredential) bool { return c == newAnchor }) } @@ -299,8 +260,7 @@ func (r *Rotator) Reconcile(set AcceptedSet, now time.Time) ReconcileResult { } r.reconcileMobileKeys(set, now) if previousMobile != newMobile && newMobile.Defined() && newMobileAlreadyAccepted { - // Primary mobile key changed to a key already in the accepted set: addCredential's gate will - // not fire for it (it's not in additions), so the caller must call ReplaceCredential itself. + // The new primary is not in additions, so addCredential's gate will not fire for it. m := newMobile result.MobilePrimaryRepoint = &m } @@ -310,39 +270,28 @@ func (r *Rotator) Reconcile(set AcceptedSet, now time.Time) ReconcileResult { return result } -// CommitAnchor atomically moves the rotator's SDK anchor pointer to the given key. The caller -// invokes this once the synchronous re-anchor sequence is ready to flip — i.e. after the new -// anchor's client is built and reports Initialized (when the anchor is a new key) or after confirming -// the existing client will be reused (when the anchor is a previously-accepted key). Until CommitAnchor -// is called, the rotator's anchor stays on the previous key so GetClient() returns the still-serving -// old client and the gate in addCredential does not fire for the pending new anchor. -// -// Aside from Initialize (which establishes the initial anchor), CommitAnchor is the only path that -// moves the anchor pointer: Reconcile deliberately does not flip it (see reconcileSDKKeys). +// CommitAnchor moves the rotator's SDK anchor pointer to key. The caller invokes it once the new +// anchor's client is ready. Until then the anchor stays on the previous key, so GetClient() returns +// the still-serving old client. Initialize and CommitAnchor are the only paths that move the pointer. func (r *Rotator) CommitAnchor(key config.SDKKey) { r.mu.Lock() defer r.mu.Unlock() r.anchorKey = key } -// RevertAnchorChange undoes the accepted-set effects of an AnchorChange whose synchronous re-anchor -// failed and rolled back. Because CommitAnchor was never called, the anchor pointer still names the -// previous anchor; this realigns the accepted set with it so the two don't disagree. +// RevertAnchorChange undoes the accepted-set effects of a failed re-anchor. CommitAnchor was never +// called, so the anchor pointer still names the previous anchor. This method realigns the accepted set. // -// - If the previous anchor is a defined key that was revoked in the same reconcile (it is no longer -// accepted — an immediate revocation rather than a grace demotion), re-admit it as a permanent -// key, since it remains the anchor and keeps serving. If it is still accepted (grace demotion), -// leave it and its expiry untouched. An undefined previous anchor (the env's first SDK key) is -// never admitted. -// - Drop the failed new anchor, but only if it was brand new; a previously-accepted key that was -// promoted and failed stays accepted as the non-anchor key it already was. +// If this reconcile revoked the previous anchor outright, RevertAnchorChange re-admits that key as a +// permanent key with no expiry. The re-admission discards the key's previous expiry, because that key +// is still the anchor and still serving. A previous anchor that is only grace-demoted stays accepted +// and keeps its expiry. The new anchor is dropped only if that key was brand new. func (r *Rotator) RevertAnchorChange(change AnchorChange) { r.mu.Lock() defer r.mu.Unlock() - // Only re-admit a defined previous anchor. When an env gains its first SDK key, the previous anchor - // is the empty (undefined) key — there is nothing to re-admit, and inserting "" would put an - // undefined credential into the accepted set (the rotator otherwise only holds defined keys). + // Only re-admit a defined previous anchor. The rotator holds defined keys only, and an env that + // gains its first SDK key has an undefined previous anchor. if change.PreviousAnchor.Defined() { if _, stillAccepted := r.acceptedSDKKeys[change.PreviousAnchor]; !stillAccepted { r.acceptedSDKKeys[change.PreviousAnchor] = AcceptedKey{} @@ -353,17 +302,16 @@ func (r *Rotator) RevertAnchorChange(change AnchorChange) { } } -// reconcilableKey constrains the generic reconcile helper to a comparable credential (so it can key a -// map) that is also an SDKCredential (so it can be logged and appended to the credential lists). +// reconcilableKey constrains the generic reconcile helper to a comparable SDKCredential, so the helper +// can both key a map with it and log it. type reconcilableKey interface { comparable SDKCredential } -// reconcileAcceptedKeys diffs the desired keys against the currently-accepted ones (SDK or mobile, -// same algorithm): a desired key not yet accepted is recorded and queued as an addition; an accepted -// key no longer desired is dropped and queued as an expiration. Per-key expiry is stored as data on -// the accepted entry; the cleanup ticker is what later acts on it. The caller must hold the write lock. +// reconcileAcceptedKeys diffs the desired keys against the currently-accepted ones, for SDK and +// mobile keys alike: a desired key not yet accepted is queued as an addition; an accepted key no +// longer desired is dropped and queued as an expiration. The caller must hold the write lock. func reconcileAcceptedKeys[K reconcilableKey]( desired map[K]AcceptedKey, accepted map[K]AcceptedKey, @@ -372,10 +320,8 @@ func reconcileAcceptedKeys[K reconcilableKey]( loggers ldlog.Loggers, kind string, ) { - // First pass: walk every key the set wants us to accept. Writing the desired entry over the - // accepted one refreshes its metadata — both the expiry and the wire "key" identifier, clearing - // the identifier when the new payload carries none so a stale name never lingers in /status. A key - // we don't yet accept is also queued as an addition. + // Writing the desired entry over the accepted one refreshes the expiry and the wire identifier. It + // clears the identifier when the new payload carries none, so a stale name never lingers in /status. for key, want := range desired { if _, ok := accepted[key]; !ok { *additions = append(*additions, key) @@ -383,9 +329,7 @@ func reconcileAcceptedKeys[K reconcilableKey]( } accepted[key] = want } - // Second pass: walk every key we currently accept and drop the ones the set no longer wants. - // Keys still desired were handled above, so skip them; the rest are revoked outright (removed - // from the map) and queued as expirations. + // Drop every accepted key the set no longer wants. Those keys are revoked outright. for key := range accepted { if _, ok := desired[key]; ok { continue @@ -396,14 +340,8 @@ func reconcileAcceptedKeys[K reconcilableKey]( } } -// reconcileSDKKeys diffs the desired SDK keys against the accepted set and applies the result via -// reconcileAcceptedKeys. The set is trusted as well-formed: BuildAcceptedSet / the builder guarantee -// the anchor is present and permanent (WithAnchor forces a nil expiry), so no special handling is -// needed here. The caller must hold the write lock. -// -// NOTE: reconcileSDKKeys does NOT flip r.anchorKey when the anchor changes. The Reconcile caller -// signals the anchor change via ReconcileResult.AnchorChange and invokes CommitAnchor to move the -// pointer once the synchronous re-anchor sequence is ready (see Reconcile + CommitAnchor). +// reconcileSDKKeys diffs the desired SDK keys against the accepted set via reconcileAcceptedKeys. The +// anchor is present and permanent, per AcceptedSetBuilder. The caller must hold the write lock. func (r *Rotator) reconcileSDKKeys(set AcceptedSet, now time.Time) { desired := make(map[config.SDKKey]AcceptedKey, len(set.sdkKeys)) for key, info := range set.sdkKeys { @@ -415,10 +353,8 @@ func (r *Rotator) reconcileSDKKeys(set AcceptedSet, now time.Time) { reconcileAcceptedKeys(desired, r.acceptedSDKKeys, &r.additions, &r.expirations, r.loggers, "SDK key") } -// reconcileMobileKeys mirrors reconcileSDKKeys for mobile keys. The set is trusted as well-formed: -// when a primary mobile key is designated, the builder guarantees it is present and permanent -// (WithPrimaryMobileKey forces a nil expiry). An empty primary means the set declared no mobile key. -// The caller must hold the lock. +// reconcileMobileKeys does for mobile keys what reconcileSDKKeys does for SDK keys. An empty primary +// means the set declared no mobile key. The caller must hold the write lock. func (r *Rotator) reconcileMobileKeys(set AcceptedSet, now time.Time) { desired := make(map[config.MobileKey]AcceptedKey, len(set.mobileKeys)) for key, info := range set.mobileKeys { @@ -444,11 +380,7 @@ func (r *Rotator) reconcileEnvironmentID(set AcceptedSet) { r.additions = append(r.additions, set.envID) } -// AcceptedKeys returns a snapshot of the full accepted credential set — all server-side SDK keys and -// all mobile keys (anchor and primary mobile key included) — grouped by kind, along with which keys -// are the designated anchor and primary mobile. The maps and the designations are read under a single -// lock so they are mutually consistent. The status endpoint maps each group to the sdkKeys[] / -// mobileKeys[] arrays. +// AcceptedKeys returns a snapshot of the full accepted set. See AcceptedKeySet. func (r *Rotator) AcceptedKeys() AcceptedKeySet { r.mu.RLock() defer r.mu.RUnlock() diff --git a/internal/envfactory/env_params.go b/internal/envfactory/env_params.go index 20d275b2..d91bf84d 100644 --- a/internal/envfactory/env_params.go +++ b/internal/envfactory/env_params.go @@ -20,21 +20,19 @@ type EnvironmentParams struct { // Identifiers contains the project and environment names and keys. Identifiers relayenv.EnvIdentifiers - // SDKKey is the environment's SDK key; if there is more than one active key, it is the latest. + // SDKKey is the environment's anchor SDK key (the wire's sdkKey.value). SDKKey config.SDKKey // MobileKey is the environment's mobile key. MobileKey config.MobileKey - // AcceptedSDKKeys is the full accepted set of SDK keys for this environment, including the - // anchor. Always non-nil after ToParams(): non-empty sdkKeys arrays populate directly; absent - // or empty sdkKeys are synthesized from the singular sdkKey field so there is always at least - // the anchor entry. + // AcceptedSDKKeys is the full accepted set of SDK keys, including the anchor. ToParams always + // leaves it non-nil, synthesizing from the singular sdkKey field when the payload has no + // sdkKeys array. AcceptedSDKKeys []AcceptedSDKKey - // AcceptedMobileKeys is the full accepted set of mobile keys for this environment. Always - // non-nil after ToParams(): non-empty mobileKeys arrays populate directly; absent or empty - // mobileKeys are synthesized from the singular mobKey field. + // AcceptedMobileKeys is the full accepted set of mobile keys. ToParams always leaves it non-nil, + // synthesizing from the singular mobKey field when the payload has no mobileKeys array. AcceptedMobileKeys []AcceptedMobileKey // TTL is the cache TTL for PHP clients. diff --git a/internal/envfactory/env_rep.go b/internal/envfactory/env_rep.go index 0b150f72..e7e357a7 100644 --- a/internal/envfactory/env_rep.go +++ b/internal/envfactory/env_rep.go @@ -14,42 +14,19 @@ import ( // file data source archive are deliberately the same. Any properties that are only used in one // or the other of those contexts should be in the appropriate package instead of here. -// EnvironmentRep is a representation of an environment that is being added or updated. +// EnvironmentRep is the wire shape of an environment, shared by RAC and the offline archive. // -// EnvironmentRep carries an environment's wire shape from RAC and the offline archive -// (same struct serves both — keep them aligned). +// Wire vocabulary: "key" is the non-secret human-readable identifier; "value" is the credential +// secret. Relay's own SDKKey, MobileKey, and SDKCredential types hold what the wire calls "value". +// Do not rename them. // -// FIELD NAMING — read this before changing anything: +// sdkKey and mobKey are the singular default credentials. sdkKey is an object because it also carries +// the legacy sdkKey.expiring slot; mobKey is a plain string because mobile keys never had one. +// sdkKeys and mobileKeys are the authoritative full accepted set, with entries of the form +// { key, value, expiry?, hasViews? }. // -// sdkKey is the singular *default* SDK key for the environment. It's an -// object ({"value": "sdk-..."}) so it can also carry the legacy -// sdkKey.expiring{value, timestamp} slot during default rotation -// (back-compat for relays predating concurrent keys). -// -// mobKey is the singular default mobile key. It's a *plain string* -// because mobile keys never had a legacy expiring slot. The shape -// asymmetry is historical, not a design choice. -// -// sdkKeys/mobileKeys are the authoritative full accepted set. Entries: -// { key: , value: , expiry?: } -// -// TERMINOLOGY: -// -// The wire "key" field is the human-readable identifier (e.g. "default-sdk"), -// non-secret — stored as AcceptedSDKKey.Key / AcceptedMobileKey.Key internally. -// The wire "value" field is the actual credential string (e.g. "sdk-xxxx-..."), -// which is the secret — stored as AcceptedSDKKey.Value / AcceptedMobileKey.Value. -// Note that relay's own types (SDKKey, MobileKey, SDKCredential) refer to what -// the wire calls "value" — they are misnamed by today's standards but stable, -// so do not rename them. -// -// Anchor selection: anchor = the sdkKeys entry whose `value` matches sdkKey.value. -// No isDefault flag — value match is the signal. -// -// Backwards compatibility: Go's default JSON decoder ignores unknown fields, so old -// relays receiving payloads with sdkKeys/mobileKeys simply ignore them and continue -// using sdkKey/mobKey. DisallowUnknownFields is intentionally not used anywhere in -// this parse path. +// This parse path never sets DisallowUnknownFields, so a relay predating concurrent keys ignores +// sdkKeys and mobileKeys and keeps using sdkKey and mobKey. type EnvironmentRep struct { EnvID config.EnvironmentID `json:"envID"` EnvKey string `json:"envKey"` @@ -99,11 +76,9 @@ type ExpiringKeyRep struct { Timestamp ldtime.UnixMillisecondTime `json:"timestamp"` } -// ConcurrentKeyRep is an entry in the sdkKeys or mobileKeys array on EnvironmentRep. -// It represents one accepted credential in an environment's concurrent key set. -// -// Key is the human-readable identifier (non-secret, e.g. "default-sdk"); Value is -// the credential secret (e.g. "sdk-xxxx-..."). See the EnvironmentRep TERMINOLOGY comment. +// ConcurrentKeyRep is an entry in the sdkKeys or mobileKeys array on EnvironmentRep. It represents one +// accepted credential in an environment's concurrent key set. See EnvironmentRep for the wire +// vocabulary. type ConcurrentKeyRep struct { Key string `json:"key"` Value string `json:"value"` @@ -146,9 +121,8 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { params.AcceptedSDKKeys = append(params.AcceptedSDKKeys, entry) } } else { - // Old-format payload: no sdkKeys array present. Synthesize AcceptedSDKKeys from the - // singular sdkKey fields so consumers always receive a consistent non-nil model - // regardless of wire format version. Key (identifier) is empty — the old format had none. + // Old-format payload: synthesize AcceptedSDKKeys from the singular sdkKey fields, so the model + // is always non-nil. The old format carried no identifier, so Key stays empty. params.AcceptedSDKKeys = make([]AcceptedSDKKey, 0, 2) params.AcceptedSDKKeys = append(params.AcceptedSDKKeys, AcceptedSDKKey{Value: r.SDKKey.Value}) if r.SDKKey.Expiring.Value.Defined() { @@ -175,9 +149,8 @@ func (r EnvironmentRep) ToParams() EnvironmentParams { } } else { // Old-format payload: synthesize from the singular mobKey field. An undefined mobKey means the - // environment has no mobile key (e.g. a server-side-only environment) — leave the set empty - // rather than synthesizing a phantom empty-value entry, which BuildAcceptedSet would otherwise - // reject as a malformed credential. + // environment has no mobile key, so leave the set empty. A phantom empty-value entry would make + // BuildAcceptedSet reject the payload. params.AcceptedMobileKeys = []AcceptedMobileKey{} if r.MobKey.Defined() { params.AcceptedMobileKeys = append(params.AcceptedMobileKeys, AcceptedMobileKey{Value: r.MobKey}) diff --git a/internal/envfactory/reconcile_helper.go b/internal/envfactory/reconcile_helper.go index 5052bcb7..07785c94 100644 --- a/internal/envfactory/reconcile_helper.go +++ b/internal/envfactory/reconcile_helper.go @@ -5,53 +5,39 @@ import ( "github.com/launchdarkly/ld-relay/v8/internal/util" ) -// BuildAcceptedSet converts an EnvironmentParams into the AcceptedSet needed by -// EnvContext.ReconcileCredentials. +// BuildAcceptedSet converts EnvironmentParams into the AcceptedSet that +// EnvContext.ReconcileCredentials needs. It is the single home for the anchor invariant, for both RAC +// and the offline archive. // -// Credential identity is keyed by value (the secret string), not by key (the human-readable -// identifier). A rename — same value, different identifier — therefore produces the same -// AcceptedSet as if nothing changed. +// Credential identity is the value (the secret), not the wire identifier, so a rename produces an +// unchanged AcceptedSet. // -// Expiry comes from AcceptedSDKKey.Expiry / AcceptedMobileKey.Expiry (the arrays). The legacy -// sdkKey.expiring wire slot is never consulted here — relay trusts the array. +// Expiry comes from the arrays. The legacy sdkKey.expiring wire slot is never consulted. // -// The anchor (params.SDKKey) is added and designated as the primary SDK key, and the primary -// mobile key (params.MobileKey) is added and designated, in addition to the full accepted arrays. -// The builder de-duplicates by value, so an anchor or primary mobile key that also appears in its -// array is added only once. +// On a structurally malformed payload it returns a *credential.MalformedCredentialSetError and an +// empty set. The caller must keep the previous accepted state, and RAC handlers must also reconnect +// the stream with jitter. // -// An error is returned (with an empty AcceptedSet) for a structurally malformed payload: an undefined -// anchor (params.SDKKey not set), a defined anchor that is absent from params.AcceptedSDKKeys, a -// defined primary mobile key (params.MobileKey) that is absent from params.AcceptedMobileKeys, a -// non-empty params.AcceptedMobileKeys with no designated primary (params.MobileKey undefined), an -// array entry with an empty value, or no usable SDK key at all. The caller must preserve the previous -// accepted state and, for RAC handlers, reconnect the stream with jitter to force a fresh put. This is -// the single home for the anchor invariant. -// -// Keys scoped to a view are filtered out here rather than at authentication time, making this the -// single funnel for both RAC and the offline archive. The second return value names the keys that were -// dropped, so callers can log what they lost. An SDK presenting one of them gets a 401: the key is -// simply absent from the lookup map. +// It filters out keys scoped to a view and names them in the second return value, so callers can log +// what they dropped. An SDK presenting one gets a 401, because the key is absent from the lookup map. func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, []string, error) { anchor := params.SDKKey b := credential.NewAcceptedSetBuilder().WithEnvironmentID(params.EnvID) var rejected []string - // Add every accepted SDK key, designating the anchor as we encounter it. WithAnchor both adds and - // designates, and forces the anchor permanent — so a payload that (wrongly) carries an expiry on - // the anchor's own entry cannot demote it. An undefined anchor never matches a (defined) array - // value, so it is never designated and Build rejects the payload. + // Add every accepted SDK key, designating the anchor on the way. WithAnchor forces the anchor + // permanent, so a payload cannot demote it with an expiry on the anchor's own entry. An undefined + // anchor never matches an array value, so Build rejects the payload. // - // Entries with an empty value are structurally malformed: relay would silently accept them but - // they can never authenticate any SDK. Reject loudly rather than produce a credential-short env. + // An entry with an empty value can never authenticate any SDK, so reject it. anchorInArray := false for _, k := range params.AcceptedSDKKeys { if !k.Value.Defined() { return credential.AcceptedSet{}, nil, credential.NewEmptyCredentialError("sdkKeys", k.Key) } switch { - // A marker on the anchor's own entry is disregarded: dropping the designated key would take the - // whole environment down, and the backend forbids views on a default key in the first place. + // A view marker on the anchor's own entry is disregarded: dropping the designated key would + // take the whole environment down, and the backend forbids views on a default key. case k.Value == anchor: anchorInArray = true b.WithAnchor(credential.SDKKeyParams{Value: k.Value, Key: util.PtrOrNil(k.Key)}) @@ -62,16 +48,14 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, []strin } } - // The anchor must be one of the accepted SDK keys: the backend lists it in sdkKeys[] (and ToParams - // synthesizes it into the array for old-format payloads). A defined anchor absent from the array is - // a structurally malformed payload — reject it. + // The anchor must be one of the accepted SDK keys: the backend lists it in sdkKeys[], and ToParams + // synthesizes it into the array for old-format payloads. if anchor.Defined() && !anchorInArray { return credential.AcceptedSet{}, nil, credential.NewAnchorNotInSetError() } - // Add every accepted mobile key, designating the primary as we encounter it. Like the anchor, - // WithPrimaryMobileKey forces the primary permanent, so an expiry the payload may carry on the - // primary's own entry cannot demote it. + // Add every accepted mobile key, designating the primary on the way. WithPrimaryMobileKey forces + // the primary permanent, as WithAnchor does for the anchor. primaryMobileInArray := false for _, k := range params.AcceptedMobileKeys { if !k.Value.Defined() { @@ -89,20 +73,14 @@ func BuildAcceptedSet(params EnvironmentParams) (credential.AcceptedSet, []strin } } - // The primary mobile key, when the environment has one, must be in mobileKeys[] — the mobile - // analogue of the anchor invariant above. A defined mobKey absent from the array is malformed: - // without this guard the primary would be silently left undesignated, clearing it on reconcile and - // breaking event forwarding. (An undefined mobKey is valid — a server-side-only environment.) + // A defined primary mobile key must be in mobileKeys[], the mobile analogue of the anchor invariant + // above. Without this guard the primary stays undesignated, which breaks event forwarding. if params.MobileKey.Defined() && !primaryMobileInArray { return credential.AcceptedSet{}, nil, credential.NewPrimaryMobileKeyNotInSetError() } - // A non-empty mobileKeys[] with no designated primary (undefined mobKey) is malformed: the reconcile - // would clear the rotator's primary mobile key without a repoint, so event forwarding would keep - // using the previous (possibly revoked) primary — silent misattribution rather than a loud - // rejection. (No mobile keys at all — empty array and undefined mobKey — stays valid: a - // server-side-only environment. Old-format payloads synthesize the array from mobKey only, so an - // undefined mobKey yields an empty array and is unaffected.) + // mobileKeys[] with no designated primary is malformed. See NewPrimaryMobileKeyMissingError. + // An empty array with an undefined mobKey stays valid: a server-side-only environment. if len(params.AcceptedMobileKeys) > 0 && !params.MobileKey.Defined() { return credential.AcceptedSet{}, nil, credential.NewPrimaryMobileKeyMissingError() } diff --git a/internal/events/event_publisher.go b/internal/events/event_publisher.go index 19c64aab..3ac6cdc3 100644 --- a/internal/events/event_publisher.go +++ b/internal/events/event_publisher.go @@ -160,11 +160,9 @@ func (o OptionCapacity) apply(p *HTTPEventPublisher) error { return nil } -// OptionInitialCapacity specifies how many events to preallocate space for in each event queue. -// The queue still grows on demand (via append) up to OptionCapacity, and events are only dropped -// once OptionCapacity is reached; this option only controls the initial allocation so that a -// publisher with a large capacity does not reserve all of that memory up front. If unset, or not -// smaller than the capacity, the full capacity is preallocated, preserving the original behavior. +// OptionInitialCapacity is how many events each queue preallocates space for. The queue still grows +// on demand up to OptionCapacity, which is where events start being dropped. If this option is unset, +// or is not smaller than the capacity, the full capacity is preallocated. type OptionInitialCapacity int //nolint:unparam // the error result is required by the OptionType interface @@ -257,10 +255,9 @@ func NewHTTPEventPublisher(authKey credential.SDKCredential, httpConfig httpconf return p, nil } -// initialQueueCapacity returns the number of events to preallocate space for in a new queue. -// It is the smaller of the configured initial capacity and the maximum capacity; when no initial -// capacity is configured (<= 0), the full maximum capacity is preallocated, which is the original -// behavior. The queue can still grow (via append) up to the maximum capacity regardless. +// initialQueueCapacity returns the number of events to preallocate space for in a new queue. It is +// the smaller of the configured initial capacity and the maximum capacity. When no initial capacity +// is configured, the full maximum capacity is preallocated. func initialQueueCapacity(capacity, initialCapacity int) int { if initialCapacity > 0 && initialCapacity < capacity { return initialCapacity @@ -271,7 +268,6 @@ func initialQueueCapacity(capacity, initialCapacity int) int { func (p *HTTPEventPublisher) append(batch eventBatch) { queue := p.queues[batch.metadata] if queue == nil { - // The queue still grows up to p.capacity via append regardless of the initial allocation. queue = &publisherQueue{events: make([]json.RawMessage, 0, initialQueueCapacity(p.capacity, p.initialCapacity))} p.queues[batch.metadata] = queue } diff --git a/internal/relayenv/env_context.go b/internal/relayenv/env_context.go index f9b5c9a1..60ab0cbc 100644 --- a/internal/relayenv/env_context.go +++ b/internal/relayenv/env_context.go @@ -40,32 +40,28 @@ type EnvContext interface { // SetIdentifiers updates the environment and project names and keys. SetIdentifiers(EnvIdentifiers) - // GetAnchorKey returns the anchor SDK key — the key that owns the upstream connection. - // Use this when you need exactly the anchor (e.g. the status endpoint's sdkKey field) rather - // than the full accepted set returned by GetCredentials. + // GetAnchorKey returns the anchor SDK key, which owns the upstream connection. Use it when you + // need exactly the anchor rather than the full accepted set that GetCredentials returns. GetAnchorKey() config.SDKKey - // GetMobileKey returns the primary (default) mobile key. Like GetAnchorKey for the anchor, use this - // for the status endpoint's mobileKey field rather than iterating GetCredentials, which may return - // several accepted mobile keys (primary + expiring) in nondeterministic order. + // GetMobileKey returns the primary (default) mobile key. GetCredentials can return several accepted + // mobile keys in nondeterministic order, so use this method where one mobile key is required. GetMobileKey() config.MobileKey - // GetAcceptedKeys returns a consistent snapshot of the full accepted credential set — all - // server-side SDK keys and all mobile keys (anchor and primary mobile key included), grouped by - // kind, plus which keys are the designated anchor and primary. The status endpoint maps each group - // to the full sdkKeys[] / mobileKeys[] arrays. + // GetAcceptedKeys returns a consistent snapshot of the full accepted set, grouped by kind. See + // credential.AcceptedKeySet. GetAcceptedKeys() credential.AcceptedKeySet - // ReconcileCredentials atomically reconciles the environment's accepted credentials to match - // newSet. The set names its own anchor (the SDK key that owns the upstream connection) and - // primary mobile key. The method owns the order of operations internally (add → re-anchor → - // remove); callers do not sequence. + // ReconcileCredentials updates the environment's accepted credentials to match newSet. Calls are + // serialized. The method owns the order of operations (add, re-anchor, remove). If the re-anchor + // fails, only the anchor change is reverted; the other changes stand. // - // newSet is assumed well-formed: it is built and validated via credential.AcceptedSetBuilder - // (which guarantees an anchor) before reaching here, so this method does not re-validate. + // newSet is assumed well-formed: credential.AcceptedSetBuilder validated it, so this method does + // not re-validate. ReconcileCredentials(newSet credential.AcceptedSet) - // GetCredentials returns all currently enabled and non-deprecated credentials for the environment. + // GetCredentials returns every credential the environment currently accepts, including keys that + // carry a future expiry. Use GetAnchorKey or GetMobileKey for the designated keys. GetCredentials() []credential.SDKCredential // GetDeprecatedCredentials returns all deprecated and not-yet-removed credentials for the environment. diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index e6c1cef8..60146096 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -101,9 +101,8 @@ type envContextImpl struct { evaluator ldeval.Evaluator eventDispatcher *events.EventDispatcher bigSegmentSync bigsegments.BigSegmentSynchronizer - // makeBigSegmentSync builds a BigSegmentSynchronizer for a given anchor SDK key, binding the - // construction-time inputs (http config, store, URIs, env ID, loggers). A re-anchor uses it to - // rebuild the synchronizer on the new anchor. nil when big segments are not configured. + // makeBigSegmentSync builds a BigSegmentSynchronizer for an anchor SDK key. A re-anchor uses it + // to rebuild the synchronizer on the new anchor. nil when big segments are not configured. makeBigSegmentSync func(anchor config.SDKKey) bigsegments.BigSegmentSynchronizer bigSegmentStore bigsegments.BigSegmentStore bigSegmentsExist bool @@ -127,17 +126,13 @@ type envContextImpl struct { offline bool closed bool - // reconcileMu serializes reconcileCredentials calls — only one runs at a time, including the - // synchronous re-anchor sequence inside it. Held separately from mu so that GetClient / GetStore / - // GetEvaluator / addCredential continue to run during the (potentially seconds-long) SDK client - // construction when re-anchoring to a new key. + // reconcileMu serializes reconcileCredentials calls, including the re-anchor sequence they run. It + // is held separately from mu so that readers keep running during the SDK client construction. reconcileMu sync.Mutex - // anchorClientGen counts how many times the upstream anchor client has been (re)established. A - // re-anchor commit bumps it. startSDKClient builds its client without c.mu, so a slow build can - // finish after a later re-anchor already installed a fresh anchor client; it captures this value at - // launch and, on completion, discards its (now stale) build if the generation has advanced rather - // than clobbering the current anchor client. Guarded by c.mu. + // anchorClientGen counts how many times the anchor client has been established. A re-anchor + // commit bumps it. startSDKClient captures it at launch and discards its build if the value + // advanced, because a slow build can finish after a later re-anchor. Guarded by c.mu. anchorClientGen uint64 } @@ -233,10 +228,8 @@ func NewEnvContext( if factory == nil { factory = bigsegments.DefaultBigSegmentSynchronizerFactory } - // Bind the construction-time inputs so a re-anchor can rebuild the synchronizer on the new anchor - // key (see reanchorBigSegmentSync). The synchronizer authenticates from the SDK key it is handed - // (bigsegments/sync sets the Authorization header from it directly), so re-anchoring only needs - // the new key; httpConfig is transport configuration and is reused as-is. + // Bind the construction-time inputs so a re-anchor can rebuild the synchronizer on the new + // anchor key (see reanchorBigSegmentSync). The synchronizer authenticates with the key it gets. baseURI := allConfig.Main.BaseURI.String() streamURI := allConfig.Main.StreamURI.String() envContext.makeBigSegmentSync = func(anchor config.SDKKey) bigsegments.BigSegmentSynchronizer { @@ -394,8 +387,7 @@ func NewEnvContext( } // Connecting may take time, so do this in parallel - // launchGen is 0 here: no re-anchor can have committed yet (the env isn't wired into reconcile until - // after construction returns), so this initial build is never superseded and its result is recorded. + // launchGen is 0 here: no re-anchor can have committed yet, so this build is never superseded. go envContext.startSDKClient(envConfig.SDKKey, readyCh, allConfig.Main.IgnoreConnectionErrors, 0) cleanupInterval := params.ExpiredCredentialCleanupInterval @@ -429,14 +421,8 @@ func (c *envContextImpl) addCredential(newCredential credential.SDKCredential) { c.registerCredentialMappings(newCredential) - // Registering the credential mappings above is all that most keys require. The one extra step is - // event forwarding for mobile keys: mobile-key event forwarding collapses to the primary mobile key, - // so a newly added mobile key repoints the event dispatcher only when it is the primary mobile key - // (a non-primary mobile key accepted in the same reconcile does not steal event forwarding). - // The upstream client lifecycle and SDK-key event repointing are deliberately not handled here: there - // is a single upstream connection per environment owned by the anchor key, and it is set up exclusively - // by construction (NewEnvContext) and moved by the re-anchor sequence (commitReanchor), which also - // repoints the SDK-key event forwarders since events collapse to the anchor per kind. + // Event forwarding collapses to the primary mobile key, so only the primary repoints the dispatcher. + // Client lifecycle and SDK-key event forwarding belong to NewEnvContext and commitReanchor. if mobileKey, ok := newCredential.(config.MobileKey); ok && mobileKey == c.keyRotator.MobileKey() { if c.eventDispatcher != nil { c.eventDispatcher.ReplaceCredential(mobileKey) @@ -466,14 +452,12 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env client, err := c.sdkClientFactory(sdkKey, c.sdkConfig, c.sdkInitTimeout) c.mu.Lock() name := c.identifiers.GetDisplayName() - // The build happens before we take c.mu. By now the env may be closed, the key may have been - // revoked, or a re-anchor may have committed a fresh anchor client since this build was launched - // (anchorClientGen advanced). In any of those cases this build is stale: close it rather than install - // it, so it cannot clobber the current anchor client. This must not rely on client==nil: a failed SDK - // build returns a non-nil, uninitialized client together with the error, so a stale failed build - // would otherwise replace a healthy anchor client with a dead one. Only defined keys are - // revocation-checked: an undefined SDK key is never tracked, and dropping its client would break envs - // that legitimately run without an SDK key (offline / not-yet-configured / tests). + // The build ran without c.mu, so it may now be stale: the env closed, the key was revoked, or a + // re-anchor advanced anchorClientGen. Close a stale build instead of installing it. + // + // The guard cannot test client == nil, because a failed build returns a non-nil uninitialized + // client. Only defined keys are revocation-checked: an undefined key is never tracked, and envs + // legitimately run without one. superseded := c.anchorClientGen != launchGen droppedInactive := false if client != nil && (c.closed || superseded || (sdkKey.Defined() && !c.sdkKeyIsActive(sdkKey))) { @@ -482,19 +466,15 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env droppedInactive = true } if client != nil { - // If a client already exists for this key (e.g. it was re-anchored back into the anchor slot - // while a prior client for it was still in its grace period), close the stale one before - // replacing it so its upstream connection and goroutines are not leaked. + // Close any stale client for this key, so its connection and goroutines are not leaked. if existing := c.clients[sdkKey]; existing != nil && existing != client { _ = existing.Close() } c.clients[sdkKey] = client c.rebuildEvaluator() // the SDK created the data store during Build; wire the evaluator to it now } - // Record this build's result as the env's init status only when it is the current anchor's build: not - // superseded by a newer anchor client, and its key is still the anchor. A genuine failure of the - // current anchor is thus recorded (the middleware 401s a broken env); a stale build's late failure is - // not, so it cannot 401 a healthy re-anchored env. + // Record this build's result as the env's init status only for the current anchor's build. A stale + // build's late failure must not 401 a healthy re-anchored env. if !superseded && sdkKey == c.keyRotator.AnchorKey() { c.initErr = err } @@ -502,10 +482,6 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env switch { case droppedInactive: - // The build finished but was superseded by a re-anchor, or its key was revoked, or the env - // closed, so it was discarded above rather than installed (even if it also errored -- a - // discarded build's error is moot). The environment is still consistent: no stale client left - // behind. c.globalLoggers.Infof("SDK key %s build was superseded, revoked, or the environment was closed "+ "before it finished initializing; the client was discarded", sdkKey.Masked()) case err != nil: @@ -528,9 +504,8 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env } } -// sdkKeyIsActive reports whether the given SDK key is still a tracked credential -- the anchor or a key -// within its deprecation grace period -- according to the rotator. startSDKClient uses this to avoid -// installing (and thereby leaking) a client for a key that was revoked while it was being built. +// sdkKeyIsActive reports whether the rotator still accepts sdkKey. startSDKClient uses this to avoid +// installing a client for a key revoked while the client was building. func (c *envContextImpl) sdkKeyIsActive(sdkKey config.SDKKey) bool { return slices.Contains(c.keyRotator.AllCredentials(), credential.SDKCredential(sdkKey)) } @@ -557,19 +532,14 @@ func (c *envContextImpl) ReconcileCredentials(newSet credential.AcceptedSet) { c.reconcileCredentials(newSet, time.Now()) } -// reconcileCredentials is the time-injectable implementation of ReconcileCredentials (now is the -// reference time for expiry math). +// reconcileCredentials is the time-injectable implementation of ReconcileCredentials. now is the +// reference time for expiry math. // -// Order: add -> re-anchor -> remove. Adding first registers the new keys' mappings; the re-anchor then -// swaps the upstream client while the old anchor is still serving, closing the old anchor's client -// once the new one is committed; removing last tears down revoked keys' mappings only once the new -// anchor is up. addCredential opens an upstream client only for the anchor -- non-anchor server keys -// are routed without a second connection. +// The order is add, re-anchor, remove. Adding first registers the new keys' mappings. The re-anchor +// then swaps the upstream client while the old anchor still serves. Removing last tears down revoked +// mappings only after the new anchor is up. addCredential never opens a client; only reanchor does. // -// reconcileMu serializes this whole method against concurrent reconciles and the cleanup ticker (see -// triggerCredentialChanges). See reanchor for the SDK-anchor swap; MobilePrimaryRepoint is handled -// inline below (a primary-mobile change to an already-accepted key isn't in additions, so addCredential -// won't repoint event forwarding for it). +// reconcileMu serializes this method against concurrent reconciles and the cleanup ticker. func (c *envContextImpl) reconcileCredentials(newSet credential.AcceptedSet, now time.Time) { c.reconcileMu.Lock() defer c.reconcileMu.Unlock() @@ -583,17 +553,15 @@ func (c *envContextImpl) reconcileCredentials(newSet credential.AcceptedSet, now if result.AnchorChange != nil { if committed := c.reanchor(result.AnchorChange); !committed { - // Rolled back: the new anchor's client never came up. Undo just this anchor change (other - // changes in the payload stand), mirroring RevertAnchorChange. A brand-new anchor had its - // mappings registered this cycle, so tear them down here; a previously-accepted anchor keeps - // its mappings and reverts to the non-anchor key it already was. + // Rolled back: the new anchor's client never came up. Undo only the anchor change; the other + // changes in this payload stand. A brand-new anchor had its mappings registered this cycle, + // so tear them down here. if !result.AnchorChange.NewAnchorPreviouslyAccepted { c.removeCredential(result.AnchorChange.NewAnchor) } c.keyRotator.RevertAnchorChange(*result.AnchorChange) - // Keep the previous anchor's client serving by not expiring it here — even if this payload - // revoked it outright. (A grace-demoted previous anchor isn't in expirations anyway, so this - // only matters for an immediate revocation.) + // Keep the previous anchor's client serving by not expiring that key here, even if this + // payload revoked it outright. previousAnchor := result.AnchorChange.PreviousAnchor expirations = slices.DeleteFunc(expirations, func(cred credential.SDKCredential) bool { return cred == previousAnchor @@ -615,31 +583,20 @@ func (c *envContextImpl) reconcileCredentials(newSet credential.AcceptedSet, now } } -// reanchor drives the synchronous re-anchor sequence for an SDK anchor change signaled by Reconcile's -// ReconcileResult.AnchorChange. Invoked by reconcileCredentials after additions have been processed -// and before expirations, so the previous anchor's client is still alive while the new client is built -// (or reused). +// reanchor moves the environment's upstream connection to change.NewAnchor. It returns true if the +// anchor was committed, and false if it rolled back (the build failed, or the env closed). +// This function is the canonical re-anchor sequence: Reconcile signals the anchor change but does not +// flip the rotator's pointer; reanchor builds or reuses the client, then commitReanchor moves it. // -// reanchor holds c.mu for the whole sequence and releases it only around the SDK client build (which -// must not hold the lock — see buildNewAnchorClient). Holding one continuous lock otherwise keeps -// Close() (which also takes c.mu) from tearing down clients or the dispatcher mid-commit, and lets -// commitReanchor assume the lock is held rather than re-acquiring it. +// Concurrency: reanchor holds c.mu for the whole sequence and releases it only around the client +// build. The continuous lock keeps Close() out of the middle of a commit, and lets commitReanchor +// assume the lock is held. // -// - When there is no existing client for the new anchor and the env is online: register its credential -// mappings if the key is brand new (Reconcile stripped it from additions), build a new SDK client, -// and on Initialized commit the anchor. On init failure, roll back: do not commit, leave the previous -// anchor authoritative (its client keeps serving), and log a structured error. -// - When a client already exists for the new anchor, or the env is offline: no build, just commit. -// (A demoted former anchor no longer has a client -- it was closed when its demotion committed -- -// so re-promoting an in-grace key builds a fresh client.) +// The new anchor needs a client only when none exists and the env is online. // -// Returns true if the anchor was committed, false if it rolled back (init failure or the env closed -// mid-build), so reconcileCredentials can back out the anchor change. On commit, the previous -// anchor's client is closed here: the anchor owns the environment's single upstream connection, and -// leaving the demoted key's client running would hold a second upstream stream feeding the same -// shared store wrapper, broadcasting every update twice to connected clients. Only the client goes; -// the demoted key's credential mappings stay registered so it keeps authenticating downstream -// connections until its grace period expires (removeCredential then finds no client to close). +// On commit, reanchor closes the previous anchor's client. Two clients would feed the same store +// wrapper and broadcast every update twice. The demoted key keeps its credential mappings, so it +// still authenticates downstream connections until its grace period expires. func (c *envContextImpl) reanchor(change *credential.AnchorChange) bool { newAnchor := change.NewAnchor previousAnchor := change.PreviousAnchor @@ -647,51 +604,40 @@ func (c *envContextImpl) reanchor(change *credential.AnchorChange) bool { c.mu.Lock() defer c.mu.Unlock() - // Two independent questions: - // - NewAnchorPreviouslyAccepted: are this key's credential mappings already registered? A brand-new - // anchor was stripped from additions, so register them now; an already-accepted key already has them. - // - the client check below: does a client already exist? If so reuse it, else build one. - // They differ for a previously-accepted non-anchor key promoted to anchor: mappings exist, client - // does not. (A live client always implies the key was already accepted, so registration never double-fires.) + // Mappings can exist without a client, but never the reverse. Reconcile stripped a brand-new anchor + // from additions, so register its mappings here; an already-accepted key has them already. if !change.NewAnchorPreviouslyAccepted { c.registerCredentialMappings(newAnchor) } - // A live client for the new anchor is reused as-is; an offline env has no upstream client to build. - // Either way there is nothing to build, so fall through to commit. + // A live client is reused as-is, and an offline env builds none. Both cases fall through to commit. why := "reused existing client" if c.clients[newAnchor] == nil { if c.offline { why = "offline — no client build" } else { - // Build the new client without the lock: sdkClientFactory can block for up to sdkInitTimeout, - // and holding c.mu that long would stall every GetClient/GetStore caller (see reconcileMu). - // reanchor's deferred Unlock releases the lock we re-acquire here on return. + // Build without the lock: sdkClientFactory can block for up to sdkInitTimeout, and holding + // c.mu that long would stall every GetClient and GetStore caller. c.mu.Unlock() client := c.buildNewAnchorClient(newAnchor, previousAnchor) c.mu.Lock() if client == nil { - // Init failed; buildNewAnchorClient already logged and closed the half-built client. Do not - // commit — leave the previous anchor authoritative. + // Init failed. buildNewAnchorClient logged the error and closed the half-built client. return false } if c.closed { - // The env was torn down while the lock was released for the build (Close() does not hold - // reconcileMu, so it can run concurrently); its client-teardown loop has already finished and - // would never close this one, so discard the freshly-built client rather than install it into - // a closed env (mirrors the guard in startSDKClient). + // Close() ran while the lock was released, and its client-teardown loop already finished, + // so it would never close this client. Discard the client instead of installing it. _ = client.Close() return false } if existing := c.clients[newAnchor]; existing != nil && existing != client { - // Stale-client guard: the lock was released for the build, so re-check and close any client - // installed concurrently for newAnchor. + // The lock was released for the build, so close any client installed concurrently. _ = existing.Close() } c.clients[newAnchor] = client - // With store handover, GetStore() returns the SAME wrapper the old client used, so the rebuilt - // evaluator serves the already-populated data immediately (no empty-store window). + // GetStore() returns the same wrapper the old client used, so there is no empty-store window. c.rebuildEvaluator() why = "built new client" } @@ -701,12 +647,9 @@ func (c *envContextImpl) reanchor(change *credential.AnchorChange) bool { return false } - // The new anchor's client is now authoritative, so tear down the previous anchor's client - // whether the key was grace-demoted or revoked outright (an undefined previous anchor has no - // entry, and a rolled-back commit never reaches here). The shared store wrapper survives: it is - // refcounted and the new anchor's client holds it. Offline mode is exempt, mirroring - // removeCredential: the offline branch above built no replacement, and the env's single - // file-data client (found by GetClient's map iteration) must keep serving across rotations. + // The new anchor's client is now authoritative, so close the previous anchor's client. The shared + // store wrapper survives: it is refcounted and the new anchor's client holds it. Offline mode is + // exempt because it built no replacement, and its single file-data client must keep serving. if !c.offline { if oldClient := c.clients[previousAnchor]; oldClient != nil { delete(c.clients, previousAnchor) @@ -716,25 +659,16 @@ func (c *envContextImpl) reanchor(change *credential.AnchorChange) bool { return true } -// buildNewAnchorClient constructs the SDK client for a re-anchor to newAnchor. It must run without c.mu -// held: sdkClientFactory can block for up to sdkInitTimeout, and holding the lock that long would stall -// every GetClient/GetStore caller. It touches only fields fixed at construction (sdkClientFactory, -// sdkConfig, sdkInitTimeout, globalLoggers), so it needs no lock — mirroring startSDKClient, which also -// builds before locking. +// buildNewAnchorClient constructs the SDK client for a re-anchor to newAnchor. It returns nil if the +// build failed, after closing any half-built client and logging the error. +// +// The caller must not hold c.mu: sdkClientFactory can block for sdkInitTimeout, which would stall +// every GetClient and GetStore caller. This method reads only construction-time fields. // -// Returns the initialized client, or nil if the build failed, in which case it has already closed any -// half-built client and logged a structured error. initErr is deliberately left untouched on failure: -// it feeds the request middleware, and setting it to the new anchor's ErrInitializationFailed would 401 -// an env that is serving fine on the previous anchor. +// It leaves initErr alone on failure. initErr feeds the request middleware, so setting it would 401 +// an env that is still serving on the previous anchor. // -// Factory contract (see sdks.ClientFactoryFunc): a factory whose construction builds the environment's -// data store must return a non-nil client even on init failure, so that the client.Close() below releases -// the store reference the build acquired (the store wrapper is refcounted — see streamUpdatesStoreWrapper). -// When client == nil there is no handle to release, and this method cannot safely release one itself: it -// has no signal for whether the store was built, so releasing unconditionally could double-release the -// still-serving previous anchor's store on an early factory error. The real SDK honors the contract (a -// failed init returns a non-nil client; an error before the store is built releases nothing); test -// factories must uphold it too. +// The Close() below relies on the store-release contract in sdks.ClientFactoryFunc. func (c *envContextImpl) buildNewAnchorClient(newAnchor, previousAnchor config.SDKKey) sdks.LDClientContext { client, err := c.sdkClientFactory(newAnchor, c.sdkConfig, c.sdkInitTimeout) if err != nil || client == nil || !client.Initialized() { @@ -751,36 +685,23 @@ func (c *envContextImpl) buildNewAnchorClient(newAnchor, previousAnchor config.S return client } -// commitReanchor is the second half of the re-anchor sequence: atomically move the rotator's anchor -// pointer, clear any stale init error now that a healthy client is current, and repoint downstream -// event/metrics forwarding. The caller must hold c.mu — reanchor holds it across the whole sequence, so -// the commit and Close() (which also takes c.mu) are mutually exclusive and Close can't tear the client -// or dispatcher out mid-commit. This mirrors addCredential, which likewise repoints event forwarding -// and reads rotator state under c.mu. -// -// Returns false without committing if the env was closed first, so callers report the rollback rather -// than a phantom success. +// commitReanchor is the second half of the re-anchor sequence: move the rotator's anchor pointer, +// clear a stale init error, and repoint event and metrics forwarding. It returns false without +// committing if the env was closed first. The caller must hold c.mu. func (c *envContextImpl) commitReanchor(newAnchor, previousAnchor config.SDKKey, why string) bool { if c.closed { - // Close() ran before we could commit. Don't flip the anchor or touch the (now-closed) dispatcher - // and metrics publisher; the env is being torn down. + // Close() ran first. The env is being torn down, so do not flip the anchor. return false } c.keyRotator.CommitAnchor(newAnchor) - // A new anchor client is now authoritative, so any startSDKClient build still in flight from before - // this commit is stale: bump the generation so it discards itself instead of clobbering this client. - // Only do this online: an offline commit (see reanchor above) installs no replacement client, so - // there is nothing for the bump to protect. Bumping anyway would strand the env's initial client - // build (launched with generation 0 at construction) if it is still in flight when this offline - // re-anchor commits: startSDKClient would see its generation superseded and discard the build, - // leaving GetClient() nil forever with no other build ever attempted. + // Any startSDKClient build still in flight is now stale, so bump the generation to make that build + // discard itself. Bump only when online: an offline commit installs no replacement client, and the + // bump would strand the env's initial build, leaving GetClient() nil forever. if !c.offline { c.anchorClientGen++ } - // The anchor now points at a healthy client (freshly built and Initialized, or a reused live - // client), so clear any init error a prior client left behind — otherwise GetInitError() and the - // request middleware would keep reporting a still-serving env as failed. + // The anchor now points at a healthy client, so clear any init error a prior client left behind. c.initErr = nil if c.metricsEventPub != nil { @@ -790,21 +711,18 @@ func (c *envContextImpl) commitReanchor(newAnchor, previousAnchor config.SDKKey, c.eventDispatcher.ReplaceCredential(newAnchor) } - // Re-wire big-segment synchronization onto the new anchor: its poll/stream requests authenticate - // with the anchor SDK key, so it must follow the anchor like the event/metrics forwarding above. + // Big-segment requests authenticate with the anchor SDK key, so the synchronizer follows the anchor. c.reanchorBigSegmentSync(newAnchor) c.globalLoggers.Infof("Re-anchored SDK from %s to %s (%s)", previousAnchor.Masked(), newAnchor.Masked(), why) return true } -// rebuildEvaluator constructs the environment's Evaluator against the current data store. It is called -// after (re)creating an SDK client, once the store is available, and is shared by the initial client -// startup and the re-anchor path. It reads and writes envContextImpl fields directly, so the caller -// must hold c.mu. +// rebuildEvaluator constructs the environment's Evaluator against the current data store. Call it +// after creating an SDK client. The caller must hold c.mu. // -// EnableSecondaryKey is set because we may evaluate for client-side SDKs sending old-style user data -// with the "secondary" attribute; it has no effect for newer SDKs that send contexts. +// EnableSecondaryKey supports client-side SDKs that send old-style user data with the "secondary" +// attribute. It has no effect for SDKs that send contexts. func (c *envContextImpl) rebuildEvaluator() { store := c.storeAdapter.GetStore() dataProvider := ldstoreimpl.NewDataStoreEvaluatorDataProvider(store, c.loggers) @@ -817,27 +735,20 @@ func (c *envContextImpl) rebuildEvaluator() { c.evaluator = ldeval.NewEvaluatorWithOptions(dataProvider, evalOptions...) } -// registerCredentialMappings wires relay's downstream-facing routing for cred: it registers the -// credential with the env's stream machinery and adds the connection→env mapping, so incoming -// SDK/client connections that authenticate with cred are served by this env. Stream handlers are built -// on demand per request in GetStreamHandler, so there is nothing per-credential to construct here. It -// does NOT start the upstream SDK client or repoint event/metrics forwarding — those are anchor-only -// concerns owned by the callers (addCredential, and the re-anchor sequence). The caller must hold c.mu. +// registerCredentialMappings registers cred with the env's stream machinery and adds the +// connection-to-env mapping, so connections that authenticate with cred reach this env. Stream +// handlers are built per request in GetStreamHandler. The caller must hold c.mu. func (c *envContextImpl) registerCredentialMappings(cred credential.SDKCredential) { c.envStreams.AddCredential(cred) c.connectionMapper.AddConnectionMapping(sdkauth.NewScoped(c.filterKey, cred), c) } -// triggerCredentialChanges drains the rotator's StepTime queue and applies the resulting additions -// and expirations. It runs on the cleanup ticker (cleanupExpiredCredentials), so it can fire at any -// moment — including while a synchronous re-anchor is in flight inside reconcileCredentials. +// triggerCredentialChanges drains the rotator's StepTime queue and applies the additions and +// expirations. It runs on the cleanup ticker, so it can fire during an in-flight re-anchor. // -// It takes reconcileMu for the whole StepTime + add/remove pass so the ticker is serialized against -// reconcileCredentials exactly the way concurrent reconciles already are. Without it, a credential -// expiry firing during an in-flight re-anchor would drain the same StepTime queue the reconcile -// relies on (the ticker could steal additions a reconcile just queued) and could removeCredential — -// closing a client — partway through the re-anchor sequence. reconcileCredentials never calls this, -// so taking reconcileMu here introduces no re-entrancy. +// It holds reconcileMu for the whole pass. Without that lock the ticker could steal additions a +// reconcile just queued, or close a client partway through a re-anchor. reconcileCredentials never +// calls this function, so there is no re-entrancy. func (c *envContextImpl) triggerCredentialChanges(now time.Time) { c.reconcileMu.Lock() defer c.reconcileMu.Unlock() @@ -874,13 +785,10 @@ func (c *envContextImpl) GetAcceptedKeys() credential.AcceptedKeySet { func (c *envContextImpl) GetClient() sdks.LDClientContext { c.mu.RLock() defer c.mu.RUnlock() - // c.clients always has at most one entry — the anchor's client. Only the anchor key triggers - // startSDKClient (in addCredential and at construction), so non-anchor server keys never open - // their own upstream connection. - // - // Offline mode uses iteration rather than key-based lookup for historical reasons; both - // approaches are correct because keyRotator is initialized with envConfig.SDKKey before - // startSDKClient is ever called. + // c.clients holds at most one entry: the anchor's client. Only construction and the re-anchor + // sequence create clients, so non-anchor server keys never open an upstream connection. Offline mode + // iterates instead of looking up by key. Both work: the rotator is initialized with envConfig.SDKKey + // before any client is built. if c.offline { for _, client := range c.clients { return client @@ -917,10 +825,8 @@ func (c *envContextImpl) GetLoggers() ldlog.Loggers { } func (c *envContextImpl) GetStreamHandler(streamProvider streams.StreamProvider, cred credential.SDKCredential) http.Handler { - // Build the handler on demand rather than storing one per (credential, provider): every handler in a - // (filter, provider) slot is identical except for the credential-derived channel id, which we resolve - // here from the request's already-authenticated credential. c.filterKey is immutable after - // construction, so this needs no lock. + // Build the handler on demand: every handler in a (filter, provider) slot differs only by the + // credential-derived channel id. c.filterKey is immutable after construction, so this needs no lock. if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, cred)); h != nil { return h } @@ -1049,10 +955,8 @@ func (c *envContextImpl) Close() error { return nil } -// consumeBigSegmentUpdates spawns a goroutine that drains sync's update channel, broadcasting a -// cache-clear + client-side invalidation for each batch. The goroutine exits when the channel closes -// (i.e. when sync is Closed). Called for the initial synchronizer and for each re-anchor replacement, so -// each synchronizer instance gets its own consumer bound to its own channel. +// consumeBigSegmentUpdates spawns a goroutine that drains sync's update channel and broadcasts a +// cache-clear and a client-side invalidation for each batch. The goroutine exits when sync closes. func (c *envContextImpl) consumeBigSegmentUpdates(sync bigsegments.BigSegmentSynchronizer) { ch := sync.SegmentUpdatesCh() if ch == nil { @@ -1060,8 +964,7 @@ func (c *envContextImpl) consumeBigSegmentUpdates(sync bigsegments.BigSegmentSyn } go func() { for range ch { - // The batch's segment keys are not needed today: we just ping all connected client-side SDKs. - // (A future evaluation-stream design would use the keys to target re-evaluation.) + // The batch's segment keys are not needed: relay pings all connected client-side SDKs. if c.sdkBigSegments != nil { c.sdkBigSegments.ClearCache() } @@ -1072,15 +975,9 @@ func (c *envContextImpl) consumeBigSegmentUpdates(sync bigsegments.BigSegmentSyn }() } -// reanchorBigSegmentSync rebuilds the big-segment synchronizer on the new anchor when the SDK anchor -// changes. The synchronizer bakes in its SDK key at construction and is not restartable, so re-anchoring -// recreates it rather than mutating it. The caller (commitReanchor) holds c.mu. -// -// If a big segment had already appeared (bigSegmentsExist -> the old synchronizer was Started), the -// replacement is Started immediately so synchronization continues without a gap. The old synchronizer is -// Closed last, which also ends its update-consumer goroutine. When big segments are not configured for -// this env there is no synchronizer and this is a no-op. sdkBigSegments (the SDK-facing store wrapper) -// persists across the re-anchor, so its polling-active state does not need re-setting here. +// reanchorBigSegmentSync rebuilds the big-segment synchronizer on the new anchor. The synchronizer +// bakes in its SDK key and is not restartable, so a re-anchor recreates it. It starts the replacement +// when the old synchronizer was started, then closes the old one. The caller holds c.mu. func (c *envContextImpl) reanchorBigSegmentSync(newAnchor config.SDKKey) { if c.bigSegmentSync == nil { return @@ -1099,12 +996,9 @@ func (c *envContextImpl) setBigSegmentsExist() { c.mu.Lock() firstTime := !c.bigSegmentsExist c.bigSegmentsExist = true - // Start the CURRENT synchronizer while holding the lock. Capturing the pointer and starting it after - // unlocking would let a concurrent re-anchor swap and Close that instance in between, so we'd Start() - // a synchronizer that was just retired. Starting c.bigSegmentSync under the lock guarantees we start - // whichever synchronizer is current -- the same one reanchorBigSegmentSync starts under this lock -- - // and never a retired one. Start() only launches a goroutine (it is non-blocking), so holding c.mu is - // fine, exactly as in reanchorBigSegmentSync. + // Start c.bigSegmentSync while holding the lock. Starting it after unlocking could start a + // synchronizer that a concurrent re-anchor already retired. Start() only launches a goroutine, so + // holding c.mu is safe. started := firstTime && c.bigSegmentSync != nil if started { c.bigSegmentSync.Start() @@ -1116,11 +1010,9 @@ func (c *envContextImpl) setBigSegmentsExist() { } } -// bigSegmentSyncConfigured reports whether this env has a big-segment synchronizer. The field's -// nil-ness is invariant over the env's life (nil iff big segments were never configured; a re-anchor -// only swaps one non-nil synchronizer for another), but the read must still be synchronized against -// that concurrent reassign in reanchorBigSegmentSync -- the store-update sink below runs on the SDK -// data-source goroutine while a re-anchor runs on the reconcile goroutine. +// bigSegmentSyncConfigured reports whether this env has a big-segment synchronizer. The read is +// synchronized against the reassign in reanchorBigSegmentSync: the store-update sink below runs on +// the SDK data-source goroutine while a re-anchor runs on the reconcile goroutine. func (c *envContextImpl) bigSegmentSyncConfigured() bool { c.mu.RLock() defer c.mu.RUnlock() diff --git a/internal/sdks/client_factory.go b/internal/sdks/client_factory.go index b2d7d3bb..74dac292 100644 --- a/internal/sdks/client_factory.go +++ b/internal/sdks/client_factory.go @@ -45,9 +45,8 @@ type DataStoreStatusInfo struct { // // Store-release contract: a factory whose client construction builds the environment's data store must // return a non-nil client even when initialization fails, so that the caller's Close() releases the -// (refcounted) store reference the build acquired. Returning (nil, err) after the store has been built -// leaks that reference — the caller has no handle to release it. Returning (nil, err) before the store -// is built is fine, as nothing was acquired. The default SDK factory honors this; test factories must too. +// refcounted store reference the build acquired. Returning (nil, err) after the store is built leaks +// that reference. Returning (nil, err) before the store is built is fine, as nothing was acquired. type ClientFactoryFunc func(sdkKey config.SDKKey, config ld.Config, timeout time.Duration) (LDClientContext, error) // LDClientConstructor is the function type of the underlying SDK client constructor. diff --git a/internal/sharedtest/configsource/rac_mock.go b/internal/sharedtest/configsource/rac_mock.go index 0760a0e4..206a07ec 100644 --- a/internal/sharedtest/configsource/rac_mock.go +++ b/internal/sharedtest/configsource/rac_mock.go @@ -41,7 +41,7 @@ func NewRACMock(t testing.TB, initialEvent *httphelpers.SSEEvent) *RACMock { } // NewRACMockWithReconnect creates a RACMock that serves firstEvent to the first client that connects -// and reconnectEvent to the next client — modeling a stream that a client restarts and reconnects to. +// and reconnectEvent to the next client, modeling a stream that a client restarts and reconnects to. // This supports malformed-payload recovery: a rejected patch forces Relay to restart // its config stream, and the backend serves a fresh, corrected put on the reconnection. // diff --git a/internal/store/relay_feature_store.go b/internal/store/relay_feature_store.go index 16ccb6ed..af139c86 100644 --- a/internal/store/relay_feature_store.go +++ b/internal/store/relay_feature_store.go @@ -21,9 +21,10 @@ import ( // Also, since streamUpdatesStoreWrapper is a wrapper for an underlying data store that could be a database, // we need to be able to specify which data store implementation is being used - also as a factory. // -// So, this factory implementation - which should only be used for a single client at a time - calls the -// wrapped factory to produce the underlying data store, then creates our own store instance, and then -// puts a reference to that instance inside itself where we can see it. +// So, this factory implementation - which is used by one environment, and hands the same wrapper to +// the incoming client on a re-anchor (see Build) - calls the wrapped factory to produce the underlying +// data store, then creates our own store instance, and then puts a reference to that instance inside +// itself where we can see it. type SSERelayDataStoreAdapter struct { store subsystems.DataStore wrappedFactory subsystems.ComponentConfigurer[subsystems.DataStore] @@ -68,13 +69,10 @@ func NewSSERelayDataStoreAdapter( // Build is called by the SDK when the LDClient is being created. // -// Store handover (re-anchor): if the adapter already holds a wrapper from -// a prior client construction, that wrapper is returned again instead of building a fresh one. This -// hands the populated, initialized data store over to the new anchor's client during a re-anchor — -// no empty-store window, no re-sync. The wrapper refcounts its holders so the underlying store is -// only torn down by the final Close (see streamUpdatesStoreWrapper.Close). If the parked wrapper has -// already been fully closed (acquire returns false), a fresh one is built rather than resurrecting a -// wrapper whose underlying store is torn down. +// Store handover: if the adapter already holds a live wrapper, Build returns that wrapper again +// instead of building a fresh one. A re-anchor therefore hands the populated store to the new anchor's +// client with no empty-store window. The wrapper refcounts its holders, so only the final Close tears +// the store down. A fully-closed wrapper (acquire returns false) is not reused. func (a *SSERelayDataStoreAdapter) Build( context subsystems.ClientContext, ) (subsystems.DataStore, error) { @@ -109,11 +107,10 @@ type streamUpdatesStoreWrapper struct { updates streams.EnvStreamUpdates loggers ldlog.Loggers - // refCount tracks how many SDK clients hold this wrapper. The first holder is implicit - // (count starts at 1 in newStreamUpdatesStoreWrapper). Each handover (Build reuse) calls - // acquire to bump the count; each client's Close decrements. The underlying store is torn - // down only when the count reaches zero, at which point closed is set so a later acquire - // refuses to hand back a wrapper whose underlying store is gone. Guarded by refMu. + // refCount tracks how many SDK clients hold this wrapper. Each handover calls acquire to bump the + // count, and each client's Close decrements it. The underlying store is torn down only when the + // count reaches zero, which also sets closed so a later acquire refuses the wrapper. Guarded by + // refMu. refMu sync.Mutex refCount int closed bool @@ -133,10 +130,8 @@ func newStreamUpdatesStoreWrapper( return relayStore } -// acquire records an additional holder of the wrapper, used by SSERelayDataStoreAdapter.Build when it -// hands this wrapper to a new client during a re-anchor. It returns false if the -// wrapper has already been fully closed (refCount reached zero and the underlying store was torn -// down); the caller must then build a fresh wrapper rather than resurrect a dead one. +// acquire records an additional holder of the wrapper. It returns false if the wrapper is already +// fully closed, in which case the caller must build a fresh wrapper. func (sw *streamUpdatesStoreWrapper) acquire() bool { sw.refMu.Lock() defer sw.refMu.Unlock() @@ -150,10 +145,9 @@ func (sw *streamUpdatesStoreWrapper) acquire() bool { func (sw *streamUpdatesStoreWrapper) Close() error { sw.refMu.Lock() if sw.closed { - // Already fully torn down. A stray extra Close (the SDK's LDClient.Close is not idempotent, so - // this depends on caller discipline) must not decrement below zero and re-satisfy the final - // guard — that would close the underlying store a second time, double-releasing a persistent - // store's connection pool. Close is idempotent past the final release. + // Already torn down. A stray extra Close must not decrement below zero and re-satisfy the + // final guard, which would close the underlying store twice and double-release a persistent + // store's connection pool. sw.refMu.Unlock() return nil } @@ -164,9 +158,8 @@ func (sw *streamUpdatesStoreWrapper) Close() error { } sw.refMu.Unlock() if !final { - // Re-anchor handover in progress: another client is still using this underlying store. - // The retiring client's Close must not tear it down — see SSERelayDataStoreAdapter.Build - // for the other half of this contract. + // Re-anchor handover in progress: another client still uses this underlying store, so the + // retiring client's Close must not tear it down. return nil } return sw.store.Close() diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index 54df1c3e..54bffecf 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -22,7 +22,7 @@ const ( ) // logViewScopedKeys reports the view-scoped credentials that BuildAcceptedSet filtered out of a -// payload. This logs once per payload that actually reaches a handler. +// payload. func logViewScopedKeys(loggers ldlog.Loggers, envName string, rejected []string) { if len(rejected) > 0 { loggers.Warnf(logMsgViewScopedKeysRejected, envName, strings.Join(rejected, ", ")) @@ -70,9 +70,7 @@ func (a *relayAutoConfigActions) UpdateEnvironment(params envfactory.Environment set, rejected, buildErr := envfactory.BuildAcceptedSet(params) if buildErr != nil { - // Credential payloads are validated at the stream parse boundary (see StreamManager) before - // being dispatched here, so a malformed set should not reach this point. Log defensively and - // preserve the previous credentials rather than applying a partial set. + // Already validated in StreamManager; keep the previous credentials if it somehow fails. a.r.loggers.Errorf(logMsgAutoConfEnvInitError, params.Identifiers.GetDisplayName(), buildErr) return } diff --git a/relay/endpoints_status.go b/relay/endpoints_status.go index 725cf26a..38cb6e0f 100644 --- a/relay/endpoints_status.go +++ b/relay/endpoints_status.go @@ -49,14 +49,11 @@ func statusHandler(relay *Relay) http.Handler { ProjName: identifiers.ProjName, } - // One consistent snapshot of the accepted credential set drives every credential field - // below — the scalar anchor/primary designations, the full sdkKeys[]/mobileKeys[] arrays, - // and expiringSdkKey — so they cannot drift relative to each other under a concurrent - // reconcile. + // One snapshot drives every credential field below, so they cannot drift against each + // other under a concurrent reconcile. accepted := clientCtx.GetAcceptedKeys() - // Scalar fields: the anchor SDK key and primary mobile key designate which array entry is - // the anchor / primary. + // The scalar fields designate which array entry is the anchor and which is the primary. if accepted.Anchor.Defined() { status.SDKKey = sdks.ObscureKey(string(accepted.Anchor)) } @@ -75,7 +72,6 @@ func statusHandler(relay *Relay) http.Handler { var expiringCandidates []expiringSDKKey for value, info := range accepted.Server { status.SDKKeys = append(status.SDKKeys, keyStatus(string(value), info)) - // expiringSdkKey considers non-anchor server keys that carry an expiry. if value != accepted.Anchor && info.Expiry != nil { expiringCandidates = append(expiringCandidates, expiringSDKKey{value: string(value), expiry: *info.Expiry}) } @@ -85,9 +81,8 @@ func statusHandler(relay *Relay) http.Handler { status.MobileKeys = append(status.MobileKeys, keyStatus(string(value), info)) } - // expiringSdkKey: the soonest-expiring non-anchor SDK key. Comparing by expiry then by value - // gives a total order, so the chosen key is deterministic even when several keys share the - // same expiry (map iteration order, and hence MinFunc's pick on a tie, is otherwise unstable). + // expiringSdkKey is the soonest-expiring non-anchor SDK key. The value comparison breaks + // ties, because map iteration order is unstable. if len(expiringCandidates) > 0 { earliest := slices.MinFunc(expiringCandidates, func(a, b expiringSDKKey) int { if c := a.expiry.Compare(b.expiry); c != 0 { @@ -189,14 +184,13 @@ func statusHandler(relay *Relay) http.Handler { } // expiringSDKKey is a candidate for the status endpoint's expiringSdkKey field: a non-anchor SDK key -// that carries an expiry. value is the plain credential; expiry is its (non-nil) expiry. +// that carries an expiry. type expiringSDKKey struct { value string expiry time.Time } -// keyStatus converts an accepted key — its credential value plus metadata — into its status-endpoint -// JSON representation, obscuring the secret value and surfacing the optional identifier and expiry. +// keyStatus converts an accepted key and its metadata into the status-endpoint JSON representation. func keyStatus(value string, k credential.AcceptedKey) api.KeyStatus { ks := api.KeyStatus{Value: sdks.ObscureKey(value)} if k.Key != nil { diff --git a/relay/filedata_actions.go b/relay/filedata_actions.go index 27008d73..d04e5140 100644 --- a/relay/filedata_actions.go +++ b/relay/filedata_actions.go @@ -60,8 +60,8 @@ func (a *relayFileDataActions) AddEnvironment(ae filedata.ArchiveEnvironment) { set, rejected, buildErr := envfactory.BuildAcceptedSet(ae.Params) if buildErr != nil { a.r.loggers.Errorf(logMsgOfflineMalformedPayload, ae.Params.Identifiers.GetDisplayName(), buildErr) - // No reconnect for offline mode: preserve previous state (env was just created with - // the singular sdkKey from envConfig) and wait for the next archive reload. + // No reconnect for offline mode, which has no live stream: keep the previous credentials and + // wait for the next archive reload. } else { logViewScopedKeys(a.r.loggers, ae.Params.Identifiers.GetDisplayName(), rejected) env.ReconcileCredentials(set) @@ -99,7 +99,7 @@ func (a *relayFileDataActions) UpdateEnvironment(ae filedata.ArchiveEnvironment) set, rejected, buildErr := envfactory.BuildAcceptedSet(ae.Params) if buildErr != nil { a.r.loggers.Errorf(logMsgOfflineMalformedPayload, ae.Params.Identifiers.GetDisplayName(), buildErr) - // Preserve previous credentials; no reconnect (offline path has no live stream). + // Keep the previous credentials. See addEnvironment: offline mode has no reconnect. } else { logViewScopedKeys(a.r.loggers, ae.Params.Identifiers.GetDisplayName(), rejected) env.ReconcileCredentials(set) From f474318be98e6f329a4a443dff0a332520e16adc Mon Sep 17 00:00:00 2001 From: "Matthew M. Keeler" Date: Mon, 17 Aug 2026 12:34:18 -0400 Subject: [PATCH 61/66] test: Remove redundant tests from the concurrent-keys suite (#827) --- .../autoconfig/stream_manager_errors_test.go | 127 +++--- .../credential/accepted_set_builder_test.go | 13 +- internal/credential/rotator_test.go | 247 +++++------- internal/envfactory/env_rep_test.go | 183 +-------- internal/envfactory/reconcile_helper_test.go | 378 +++++------------- internal/events/event_publisher_test.go | 12 - ...v_context_credential_serialization_test.go | 62 --- internal/relayenv/env_context_impl_test.go | 114 +----- .../env_context_reanchor_behavior_test.go | 54 +-- .../env_context_reanchor_bigsegment_test.go | 78 +--- .../env_context_reanchor_helpers_test.go | 6 - .../env_context_reanchor_rollback_test.go | 71 ---- .../env_context_reanchor_synchronous_test.go | 91 +---- .../store_handover_realclient_test.go | 90 +---- .../store/store_rebuild_after_close_test.go | 66 --- internal/store/store_refcount_test.go | 32 ++ relay/concurrent_keys_auth_test.go | 316 ++++----------- relay/concurrent_keys_lifecycle_test.go | 132 +++--- relay/concurrent_keys_views_test.go | 140 ++----- relay/endpoints_status_test.go | 29 -- relay/relay_endpoints_test.go | 14 +- 21 files changed, 516 insertions(+), 1739 deletions(-) delete mode 100644 internal/relayenv/env_context_credential_serialization_test.go delete mode 100644 internal/relayenv/env_context_reanchor_rollback_test.go delete mode 100644 internal/store/store_rebuild_after_close_test.go diff --git a/internal/autoconfig/stream_manager_errors_test.go b/internal/autoconfig/stream_manager_errors_test.go index 56598cb3..19fdcd1d 100644 --- a/internal/autoconfig/stream_manager_errors_test.go +++ b/internal/autoconfig/stream_manager_errors_test.go @@ -34,81 +34,59 @@ func eventShouldCauseStreamRestart(t *testing.T, event httphelpers.SSEEvent) { } // A credential payload that is valid JSON and a structurally valid event, but whose credential set -// cannot be built (e.g. an undefined anchor SDK key, or mobile keys with no designated primary), must -// be caught at the parse boundary: the previous state is preserved (no AddEnvironment/UpdateEnvironment -// dispatched) and the stream is restarted so the backend resends a fresh put. This is -// verified for both patch and put, since both paths run the validation before the version is recorded. +// cannot be built (here, an undefined anchor SDK key), must be caught at the parse boundary: the +// previous state is preserved (no AddEnvironment/UpdateEnvironment dispatched) and the stream is +// restarted so the backend resends a fresh put. This is verified for both patch and put, since both +// paths run the validation before the version is recorded. +// +// One rejection shape is enough here: this code path is identical for any non-nil error out of +// BuildAcceptedSet. The individual shapes are enumerated at their source, by +// envfactory.TestBuildAcceptedSet_MalformedPayloads. func TestMalformedCredentialPayloadCausesStreamRestart(t *testing.T) { - // Each shape is a distinct way BuildAcceptedSet rejects a structurally malformed payload; all ride - // the same malformed-payload machinery. - malformedShapes := []struct { - name string - make func(envfactory.EnvironmentRep) envfactory.EnvironmentRep - }{ - { - name: "undefined anchor SDK key", - make: func(env envfactory.EnvironmentRep) envfactory.EnvironmentRep { - env.SDKKey = envfactory.SDKKeyRep{Value: config.SDKKey("")} - return env - }, - }, - { - name: "mobile keys without a designated primary", - make: func(env envfactory.EnvironmentRep) envfactory.EnvironmentRep { - env.MobKey = config.MobileKey("") // no primary designated... - env.MobileKeys = []envfactory.ConcurrentKeyRep{{Key: "mob-1", Value: "mobkey1"}} // ...but non-empty - return env - }, - }, - } + malformedEnv := testEnv1 + malformedEnv.SDKKey = envfactory.SDKKeyRep{Value: config.SDKKey("")} // undefined anchor - for _, shape := range malformedShapes { - t.Run(shape.name, func(t *testing.T) { - malformedEnv := shape.make(testEnv1) - - t.Run("patch", func(t *testing.T) { - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - <-p.requestsCh - p.stream.Enqueue(makePatchEnvEvent(malformedEnv)) - select { - case m := <-p.messageHandler.received: - require.Failf(t, "unexpected message", - "must not dispatch for a malformed payload, got %s", m) - case <-p.requestsCh: // reconnect request == stream restart - p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") - case <-time.After(time.Second): - require.Fail(t, "timed out waiting for stream restart") - } - }) - }) + t.Run("patch", func(t *testing.T) { + streamManagerTest(t, nil, func(p streamManagerTestParams) { + p.startStream() + <-p.requestsCh + p.stream.Enqueue(makePatchEnvEvent(malformedEnv)) + select { + case m := <-p.messageHandler.received: + require.Failf(t, "unexpected message", + "must not dispatch for a malformed payload, got %s", m) + case <-p.requestsCh: // reconnect request == stream restart + p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") + case <-time.After(time.Second): + require.Fail(t, "timed out waiting for stream restart") + } + }) + }) - t.Run("put", func(t *testing.T) { - streamManagerTest(t, nil, func(p streamManagerTestParams) { - p.startStream() - <-p.requestsCh - p.stream.Enqueue(makeEnvPutEvent(malformedEnv)) - // The malformed env is skipped (no add/update); a put still reports ReceivedAllEnvironments, - // which we tolerate. We require that the stream restarts and that no add/update is dispatched. - deadline := time.After(2 * time.Second) - for { - select { - case m := <-p.messageHandler.received: - if m.add != nil || m.update != nil { - require.Failf(t, "unexpected message", - "must not dispatch add/update for a malformed payload, got %s", m) - } - case <-p.requestsCh: // reconnect request == stream restart - p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") - return - case <-deadline: - require.Fail(t, "timed out waiting for stream restart") - } + t.Run("put", func(t *testing.T) { + streamManagerTest(t, nil, func(p streamManagerTestParams) { + p.startStream() + <-p.requestsCh + p.stream.Enqueue(makeEnvPutEvent(malformedEnv)) + // The malformed env is skipped (no add/update); a put still reports ReceivedAllEnvironments, + // which we tolerate. We require that the stream restarts and that no add/update is dispatched. + deadline := time.After(2 * time.Second) + for { + select { + case m := <-p.messageHandler.received: + if m.add != nil || m.update != nil { + require.Failf(t, "unexpected message", + "must not dispatch add/update for a malformed payload, got %s", m) } - }) - }) + case <-p.requestsCh: // reconnect request == stream restart + p.mockLog.AssertMessageMatch(t, true, ldlog.Error, "malformed credential payload") + return + case <-deadline: + require.Fail(t, "timed out waiting for stream restart") + } + } }) - } + }) } // A put carrying malformed credential payloads must not corrupt the persistent cache: valid envs are @@ -274,15 +252,6 @@ func TestMalformedCredentialPayloadRecoversAfterReconnect(t *testing.T) { malformed.SDKKey = envfactory.SDKKeyRep{Value: config.SDKKey("")} // undefined anchor malformedRecoveryTest(t, makePatchEnvEvent(malformed), correctedEnv) }) - - t.Run("anchor defined but absent from sdkKeys[] is malformed and recovers", func(t *testing.T) { - // A defined anchor (sdkKey.value) that does not appear in the authoritative sdkKeys[] array is - // treated as malformed by BuildAcceptedSet, just like an undefined anchor — exercise that variant - // through the stream manager. - malformed := testEnv1 - malformed.SDKKeys = []envfactory.ConcurrentKeyRep{{Key: "other", Value: "sdk-other-value"}} - malformedRecoveryTest(t, makeEnvPutEvent(malformed), correctedEnv) - }) } func TestMalformedJSONInEventCausesStreamRestart(t *testing.T) { diff --git a/internal/credential/accepted_set_builder_test.go b/internal/credential/accepted_set_builder_test.go index 3b7b6be4..c32b226b 100644 --- a/internal/credential/accepted_set_builder_test.go +++ b/internal/credential/accepted_set_builder_test.go @@ -2,6 +2,7 @@ package credential import ( "testing" + "time" "github.com/launchdarkly/ld-relay/v8/config" @@ -30,18 +31,26 @@ func TestAcceptedSetBuilderValidation(t *testing.T) { } func TestAcceptedSetBuilderDeduplicates(t *testing.T) { - // Adding the same key more than once (including via WithPrimary*) keeps a single entry. + // Adding the same key more than once (including via WithPrimary*) keeps a single entry, and a + // WithPrimary* designation overwrites whatever metadata an earlier plain add recorded: a mobile key + // first listed with an expiry and then designated primary ends up permanent. That is what keeps the + // designated primary from ever being reported as torn down, mirroring the SDK anchor. It is a builder + // contract rather than a Reconcile behaviour — BuildAcceptedSet's switch is exclusive, so this + // interleaving is unreachable from the wire — which is why it is pinned here. + pastExpiry := time.Unix(1000, 0) set := mustBuild(t, NewAcceptedSetBuilder(). WithSDKKey(SDKKeyParams{Value: "sdk"}). WithAnchor(SDKKeyParams{Value: "sdk"}). WithSDKKey(SDKKeyParams{Value: "sdk"}). - WithMobileKey(MobileKeyParams{Value: "mob"}). + WithMobileKey(MobileKeyParams{Value: "mob", Expiry: &pastExpiry}). WithPrimaryMobileKey(MobileKeyParams{Value: "mob"})) assert.Len(t, set.sdkKeys, 1) assert.Len(t, set.mobileKeys, 1) assert.Equal(t, config.SDKKey("sdk"), set.anchor) assert.Equal(t, config.MobileKey("mob"), set.primaryMobileKey) + assert.Nil(t, set.mobileKeys[config.MobileKey("mob")].Expiry, + "the designated primary mobile key is always permanent, overwriting a prior entry's expiry") } // mustBuild builds the set and fails the test if validation rejects it. It is shared by the builder diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 3962a690..5103a73c 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -15,12 +15,6 @@ func newTestRotator() *Rotator { return NewRotator(ldlogtest.NewMockLog().Loggers) } -func TestNewRotator(t *testing.T) { - mockLog := ldlogtest.NewMockLog() - rotator := NewRotator(mockLog.Loggers) - assert.NotNil(t, rotator) -} - func TestInitializePopulatesAcceptedSets(t *testing.T) { mockLog := ldlogtest.NewMockLog() rotator := NewRotator(mockLog.Loggers) @@ -49,61 +43,75 @@ func TestInitializePopulatesAcceptedSets(t *testing.T) { assert.Equal(t, envID, rotator.EnvironmentID()) } -func TestReconcileAnchorOnly(t *testing.T) { - r := newTestRotator() - anchor := config.SDKKey("anchor") - now := time.Now() - - result := r.Reconcile(mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor})), now) - require.NotNil(t, result.AnchorChange, "anchor transition from empty to defined is signaled") - r.CommitAnchor(result.AnchorChange.NewAnchor) - additions, expirations := r.StepTime(now) - - // The anchor is stripped from additions — the synchronous re-anchor sequence owns its setup. - assert.Empty(t, additions) - assert.Empty(t, expirations) - assert.Equal(t, anchor, r.AnchorKey()) - assert.ElementsMatch(t, []SDKCredential{anchor}, r.AllCredentials()) - assert.Empty(t, r.DeprecatedCredentials()) -} - -func TestReconcileMultipleSDKKeys(t *testing.T) { - r := newTestRotator() +// TestReconcileDiff covers the additions/expirations diff Reconcile produces for the basic accepted-set +// shapes, plus what each shape leaves in the rotator's public view. The anchor is always stripped from +// additions — the synchronous re-anchor sequence in env_context_impl owns its setup — so a lone anchor +// yields empty additions. +func TestReconcileDiff(t *testing.T) { anchor := config.SDKKey("anchor") other := config.SDKKey("other") - now := time.Now() - - result := r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor}).WithSDKKey(SDKKeyParams{Value: other})), now) - require.NotNil(t, result.AnchorChange) - r.CommitAnchor(result.AnchorChange.NewAnchor) - additions, expirations := r.StepTime(now) - - // Both server keys are accepted; only the non-anchor server key is in additions (the anchor is - // owned by the synchronous re-anchor sequence in env_context_impl). - assert.ElementsMatch(t, []SDKCredential{other}, additions) - assert.Empty(t, expirations) - assert.Equal(t, anchor, r.AnchorKey()) - assert.ElementsMatch(t, []SDKCredential{anchor, other}, r.AllCredentials()) - assert.Empty(t, r.DeprecatedCredentials()) -} - -func TestReconcileMultipleMobileKeys(t *testing.T) { - r := newTestRotator() - anchor := config.SDKKey("anchor") mob1 := config.MobileKey("mob1") mob2 := config.MobileKey("mob2") - now := time.Now() - r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor}).WithPrimaryMobileKey(MobileKeyParams{Value: mob1}).WithMobileKey(MobileKeyParams{Value: mob2})), now) - additions, _ := r.StepTime(now) + tests := []struct { + name string + build func() *AcceptedSetBuilder + wantAdditions []SDKCredential + wantAllCredentials []SDKCredential + wantMobileKey config.MobileKey + }{ + { + // The distinguishing case: a lone anchor produces NO additions at all. + name: "anchor only", + build: func() *AcceptedSetBuilder { return NewAcceptedSetBuilder().WithAnchor(SDKKeyParams{Value: anchor}) }, + wantAdditions: nil, + wantAllCredentials: []SDKCredential{anchor}, + }, + { + // Both server keys are accepted; only the non-anchor one is a fresh addition. + name: "multiple SDK keys", + build: func() *AcceptedSetBuilder { + return NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: other}) + }, + wantAdditions: []SDKCredential{other}, + wantAllCredentials: []SDKCredential{anchor, other}, + }, + { + // Every mobile key is a fresh addition, the designated primary included — only the SDK anchor + // is stripped. + name: "multiple mobile keys", + build: func() *AcceptedSetBuilder { + return NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob1}). + WithMobileKey(MobileKeyParams{Value: mob2}) + }, + wantAdditions: []SDKCredential{mob1, mob2}, + wantAllCredentials: []SDKCredential{anchor, mob1, mob2}, + wantMobileKey: mob1, + }, + } - // Every mobile key is accepted; the anchor is owned by the synchronous re-anchor (stripped from - // additions). The designated primary mobile key and the other mobile key remain in additions. - assert.ElementsMatch(t, []SDKCredential{mob1, mob2}, additions) - assert.Equal(t, mob1, r.MobileKey()) - assert.ElementsMatch(t, []SDKCredential{anchor, mob1, mob2}, r.AllCredentials()) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := newTestRotator() + now := time.Now() + + result := r.Reconcile(mustBuild(t, tt.build()), now) + require.NotNil(t, result.AnchorChange, "anchor transition from empty to defined is signaled") + r.CommitAnchor(result.AnchorChange.NewAnchor) + additions, expirations := r.StepTime(now) + + assert.ElementsMatch(t, tt.wantAdditions, additions) + assert.Empty(t, expirations) + assert.Equal(t, anchor, r.AnchorKey()) + assert.Equal(t, tt.wantMobileKey, r.MobileKey()) + assert.ElementsMatch(t, tt.wantAllCredentials, r.AllCredentials()) + assert.Empty(t, r.DeprecatedCredentials()) + }) + } } func TestReconcileRevokesOmittedKeys(t *testing.T) { @@ -126,63 +134,12 @@ func TestReconcileRevokesOmittedKeys(t *testing.T) { assert.ElementsMatch(t, []SDKCredential{anchor}, r.AllCredentials()) } -func TestReconcileAcceptsExpiringKeysAsData(t *testing.T) { - // Reconcile stores per-key expiry as data on the accepted entry; before that expiry passes, an - // expiring key is still accepted (it authenticates and appears in AllCredentials) while also - // being reported as deprecated — accepted, but on its way out. The cleanup ticker (StepTime) only - // drops it once the expiry elapses — see TestReconcileExpiringKeysAreEvictedByStepTime. - r := newTestRotator() - anchor := config.SDKKey("anchor") - expiringSDK := config.SDKKey("expiring-sdk") - mob := config.MobileKey("mob") - expiringMobile := config.MobileKey("expiring-mob") - now := time.Unix(1000, 0) - - r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(SDKKeyParams{Value: anchor}). - WithSDKKey(SDKKeyParams{Value: expiringSDK, Expiry: util.PtrOrNil(now.Add(time.Hour))}). - WithPrimaryMobileKey(MobileKeyParams{Value: mob}). - WithMobileKey(MobileKeyParams{Value: expiringMobile, Expiry: util.PtrOrNil(now.Add(time.Hour))})), - now) - additions, expirations := r.StepTime(now) - - // Anchor is stripped from additions (owned by the synchronous re-anchor); other keys flow through. - assert.ElementsMatch(t, []SDKCredential{expiringSDK, mob, expiringMobile}, additions) - assert.Empty(t, expirations) - // Every key is accepted (still authenticates)... - assert.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, r.AllCredentials()) - // ...and the non-anchor SDK key carrying an expiry is also reported as deprecated (being phased - // out). The expiring mobile key is not: there is no expiringMobileKey status field, so the reconcile - // path treats it as accepted-only. - assert.ElementsMatch(t, []SDKCredential{expiringSDK}, r.DeprecatedCredentials()) -} - -func TestReconcilePrimaryMobileKeyIsAlwaysAccepted(t *testing.T) { - // Defensive: even if the designated primary mobile key is also listed with a past expiry, it must - // stay accepted (mirroring the SDK anchor), so AllCredentials never reports a torn-down key. - r := newTestRotator() - anchor := config.SDKKey("anchor") - mob := config.MobileKey("mob") - now := time.Unix(1000, 0) - - set := mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(SDKKeyParams{Value: anchor}). - WithMobileKey(MobileKeyParams{Value: mob, Expiry: util.PtrOrNil(now.Add(-time.Hour))}). // already expired in the payload... - WithPrimaryMobileKey(MobileKeyParams{Value: mob})) // ...but designated as the primary - r.Reconcile(set, now) - r.StepTime(now) - - assert.Equal(t, mob, r.MobileKey()) - assert.Contains(t, r.AllCredentials(), SDKCredential(mob)) - _, accepted := r.acceptedMobileKeys[mob] - assert.True(t, accepted, "the primary mobile key must remain in the accepted set") -} - func TestReconcileExpiringKeysAreEvictedByStepTime(t *testing.T) { // End-to-end on the reconcile path: a reconcile records per-key expiry as data on the accepted - // entry, and the cleanup ticker (StepTime) later drops both the expiring SDK key and the expiring - // mobile key once their expiry elapses. The anchor and primary mobile key carry no expiry and survive. + // entry, so before the expiry passes an expiring key is still accepted (it authenticates and appears + // in AllCredentials) while also being reported as deprecated — accepted, but on its way out. The + // cleanup ticker (StepTime) later drops both the expiring SDK key and the expiring mobile key once + // their expiry elapses. The anchor and primary mobile key carry no expiry and survive. r := newTestRotator() anchor := config.SDKKey("anchor") expiringSDK := config.SDKKey("expiring-sdk") @@ -203,6 +160,13 @@ func TestReconcileExpiringKeysAreEvictedByStepTime(t *testing.T) { require.ElementsMatch(t, []SDKCredential{expiringSDK, mob, expiringMobile}, additions) require.Empty(t, expirations) + // Every key is accepted (still authenticates)... + require.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile}, r.AllCredentials()) + // ...and the non-anchor SDK key carrying an expiry is also reported as deprecated (being phased + // out). The expiring mobile key is not: there is no expiringMobileKey status field, so the reconcile + // path treats it as accepted-only. + require.ElementsMatch(t, []SDKCredential{expiringSDK}, r.DeprecatedCredentials()) + // At the exact expiry, expiry is strict (now must be strictly after), so nothing is dropped yet. additions, expirations = r.StepTime(expiry) assert.Empty(t, additions) @@ -217,55 +181,42 @@ func TestReconcileExpiringKeysAreEvictedByStepTime(t *testing.T) { assert.NotContains(t, r.AllCredentials(), SDKCredential(expiringMobile)) } -func TestReconcileAlreadyExpiredKeyIsIgnoredOnAdd(t *testing.T) { - // An entry in the reconcile payload whose expiry is already in the past is treated as absent — - // the reconcile path filters it before calling reconcileAcceptedKeys, so it is never added. - r := newTestRotator() - anchor := config.SDKKey("anchor") - staleKey := config.SDKKey("stale") - now := time.Unix(2000, 0) - alreadyExpired := now.Add(-time.Hour) - - result := r.Reconcile( - mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(SDKKeyParams{Value: anchor}). - WithSDKKey(SDKKeyParams{Value: staleKey, Expiry: util.PtrOrNil(alreadyExpired)})), - now) - require.NotNil(t, result.AnchorChange) - r.CommitAnchor(result.AnchorChange.NewAnchor) - additions, expirations := r.StepTime(now) - - // The fresh anchor is stripped from additions (the synchronous re-anchor sequence owns its setup), - // and the already-expired stale key is never accepted — so nothing is added. - assert.Empty(t, additions) - assert.Empty(t, expirations) - assert.ElementsMatch(t, []SDKCredential{anchor}, r.AllCredentials()) -} - func TestReconcileExpiryBoundaryIsStrictlyAfter(t *testing.T) { // The reconcile-side filter that treats an already-expired key as absent must honor the same // strictly-after contract as StepTime (see the doc comment on StepTime): a key whose expiry lands // exactly on `now` is still accepted by Reconcile, and only becomes absent once `now` is one instant - // past the expiry. + // past the expiry. An entry filtered out this way is never added at all — it is not queued as an + // addition for the caller to wire up and then immediately tear down. anchor := config.SDKKey("anchor") staleKey := config.SDKKey("stale") expiry := time.Unix(2000, 0) - atBoundary := newTestRotator() - atBoundary.Reconcile( - mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(SDKKeyParams{Value: anchor}). - WithSDKKey(SDKKeyParams{Value: staleKey, Expiry: util.PtrOrNil(expiry)})), - expiry) + reconcileAt := func(now time.Time) (*Rotator, []SDKCredential, []SDKCredential) { + r := newTestRotator() + result := r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: staleKey, Expiry: util.PtrOrNil(expiry)})), + now) + require.NotNil(t, result.AnchorChange) + r.CommitAnchor(result.AnchorChange.NewAnchor) + additions, expirations := r.StepTime(now) + return r, additions, expirations + } + + atBoundary, additions, expirations := reconcileAt(expiry) assert.Contains(t, atBoundary.AllCredentials(), SDKCredential(staleKey), "a key expiring exactly at now is still accepted by Reconcile") + // The anchor is stripped from additions (the synchronous re-anchor sequence owns its setup), so the + // still-accepted stale key is the only addition. + assert.ElementsMatch(t, []SDKCredential{staleKey}, additions) + assert.Empty(t, expirations) - pastBoundary := newTestRotator() - pastBoundary.Reconcile( - mustBuild(t, NewAcceptedSetBuilder(). - WithAnchor(SDKKeyParams{Value: anchor}). - WithSDKKey(SDKKeyParams{Value: staleKey, Expiry: util.PtrOrNil(expiry)})), - expiry.Add(1*time.Millisecond)) + pastBoundary, additions, expirations := reconcileAt(expiry.Add(1 * time.Millisecond)) assert.NotContains(t, pastBoundary.AllCredentials(), SDKCredential(staleKey), "a key one instant past its expiry is treated as absent by Reconcile") + assert.ElementsMatch(t, []SDKCredential{anchor}, pastBoundary.AllCredentials()) + // Filtered before reconcileAcceptedKeys ever sees it: nothing to add, nothing to expire. + assert.Empty(t, additions) + assert.Empty(t, expirations) } func TestReconcileDeExpiryRestoresKey(t *testing.T) { diff --git a/internal/envfactory/env_rep_test.go b/internal/envfactory/env_rep_test.go index 6baaea75..87a89478 100644 --- a/internal/envfactory/env_rep_test.go +++ b/internal/envfactory/env_rep_test.go @@ -110,7 +110,13 @@ func TestEnvironmentRepJSONFormat(t *testing.T) { } // TestEnvironmentRepNewFormatWithArrays parses a realistic RAC put payload carrying the new -// sdkKeys/mobileKeys arrays and verifies the struct fields are populated correctly. +// sdkKeys/mobileKeys arrays and verifies the struct fields are populated correctly and carried through +// ToParams onto the accepted entries. +// +// It also pins the hasViews wire contract on both arrays. The absent-field case is the important one: +// hasViews is a plain bool, so an entry that omits it — every entry a backend that predates the field +// emits — decodes to false and is treated as not view-scoped. An explicit false is indistinguishable +// from absent, which is the intent: there is no third state. func TestEnvironmentRepNewFormatWithArrays(t *testing.T) { expiryMs := int64(1700000000000) jsonStr := `{ @@ -123,10 +129,12 @@ func TestEnvironmentRepNewFormatWithArrays(t *testing.T) { "sdkKey": { "value": "sdk-anchor" }, "sdkKeys": [ { "key": "default-sdk", "value": "sdk-anchor" }, - { "key": "service-a", "value": "sdk-service-a", "expiry": 1700000000000 } + { "key": "service-a", "value": "sdk-service-a", "expiry": 1700000000000, "hasViews": false }, + { "key": "view-scoped", "value": "sdk-viewy", "hasViews": true } ], "mobileKeys": [ - { "key": "mob-key-1", "value": "mob-f41c" } + { "key": "mob-key-1", "value": "mob-f41c" }, + { "key": "view-scoped-mob", "value": "mob-viewy", "hasViews": true } ], "secureMode": false, "version": 26 @@ -138,73 +146,28 @@ func TestEnvironmentRepNewFormatWithArrays(t *testing.T) { assert.Equal(t, config.SDKKey("sdk-anchor"), rep.SDKKey.Value) assert.Equal(t, config.MobileKey("mob-f41c"), rep.MobKey) - require.Len(t, rep.SDKKeys, 2) + require.Len(t, rep.SDKKeys, 3) assert.Equal(t, ConcurrentKeyRep{Key: "default-sdk", Value: "sdk-anchor"}, rep.SDKKeys[0]) assert.Equal(t, ConcurrentKeyRep{Key: "service-a", Value: "sdk-service-a", Expiry: &expiryMs}, rep.SDKKeys[1]) + assert.Equal(t, ConcurrentKeyRep{Key: "view-scoped", Value: "sdk-viewy", HasViews: true}, rep.SDKKeys[2]) + // An absent hasViews and an explicit false both decode to false, indistinguishably. + assert.False(t, rep.SDKKeys[0].HasViews, "an absent hasViews must decode to false") + assert.False(t, rep.SDKKeys[1].HasViews, "an explicit false is indistinguishable from absent") - require.Len(t, rep.MobileKeys, 1) + require.Len(t, rep.MobileKeys, 2) assert.Equal(t, ConcurrentKeyRep{Key: "mob-key-1", Value: "mob-f41c"}, rep.MobileKeys[0]) + assert.Equal(t, ConcurrentKeyRep{Key: "view-scoped-mob", Value: "mob-viewy", HasViews: true}, rep.MobileKeys[1]) + assert.False(t, rep.MobileKeys[0].HasViews) params := rep.ToParams() - require.Len(t, params.AcceptedSDKKeys, 2) + require.Len(t, params.AcceptedSDKKeys, 3) assert.Equal(t, AcceptedSDKKey{Key: "default-sdk", Value: config.SDKKey("sdk-anchor")}, params.AcceptedSDKKeys[0]) assert.Equal(t, AcceptedSDKKey{ Key: "service-a", Value: config.SDKKey("sdk-service-a"), Expiry: time.UnixMilli(expiryMs), }, params.AcceptedSDKKeys[1]) - - require.Len(t, params.AcceptedMobileKeys, 1) - assert.Equal(t, AcceptedMobileKey{Key: "mob-key-1", Value: config.MobileKey("mob-f41c")}, params.AcceptedMobileKeys[0]) -} - -// TestEnvironmentRepViewScopedKeys pins the hasViews wire contract on both arrays: it decodes onto -// ConcurrentKeyRep and is carried through ToParams onto the accepted entries, where BuildAcceptedSet -// consumes it. -// -// The absent-field case is the important one. hasViews is a plain bool, so an entry that omits it — -// every entry a backend that predates the field emits — decodes to false and is treated as not -// view-scoped. An explicit false is indistinguishable from absent, which is the intent: there is no -// third state. -func TestEnvironmentRepViewScopedKeys(t *testing.T) { - jsonStr := `{ - "envID": "68e5179e8307e4099c277e2a", - "envKey": "production", - "envName": "Production", - "mobKey": "mob-primary", - "projKey": "my-project", - "projName": "My Project", - "sdkKey": { "value": "sdk-anchor" }, - "sdkKeys": [ - { "key": "default-sdk", "value": "sdk-anchor" }, - { "key": "service-a", "value": "sdk-service-a", "hasViews": false }, - { "key": "view-scoped", "value": "sdk-viewy", "hasViews": true } - ], - "mobileKeys": [ - { "key": "default-mob", "value": "mob-primary" }, - { "key": "view-scoped-mob", "value": "mob-viewy", "hasViews": true } - ], - "version": 26 - }` - - var rep EnvironmentRep - require.NoError(t, json.Unmarshal([]byte(jsonStr), &rep)) - - require.Len(t, rep.SDKKeys, 3) - assert.False(t, rep.SDKKeys[0].HasViews, "an absent hasViews must decode to false") - assert.False(t, rep.SDKKeys[1].HasViews) - assert.True(t, rep.SDKKeys[2].HasViews) - - require.Len(t, rep.MobileKeys, 2) - assert.False(t, rep.MobileKeys[0].HasViews) - assert.True(t, rep.MobileKeys[1].HasViews) - - params := rep.ToParams() - - require.Len(t, params.AcceptedSDKKeys, 3) - assert.Equal(t, AcceptedSDKKey{Key: "default-sdk", Value: config.SDKKey("sdk-anchor")}, params.AcceptedSDKKeys[0]) - assert.Equal(t, AcceptedSDKKey{Key: "service-a", Value: config.SDKKey("sdk-service-a")}, params.AcceptedSDKKeys[1]) assert.Equal(t, AcceptedSDKKey{ Key: "view-scoped", Value: config.SDKKey("sdk-viewy"), @@ -212,114 +175,10 @@ func TestEnvironmentRepViewScopedKeys(t *testing.T) { }, params.AcceptedSDKKeys[2]) require.Len(t, params.AcceptedMobileKeys, 2) - assert.Equal(t, AcceptedMobileKey{Key: "default-mob", Value: config.MobileKey("mob-primary")}, params.AcceptedMobileKeys[0]) + assert.Equal(t, AcceptedMobileKey{Key: "mob-key-1", Value: config.MobileKey("mob-f41c")}, params.AcceptedMobileKeys[0]) assert.Equal(t, AcceptedMobileKey{ Key: "view-scoped-mob", Value: config.MobileKey("mob-viewy"), HasViews: true, }, params.AcceptedMobileKeys[1]) } - -// TestEnvironmentRepOldFormatNoArrays verifies that an old-format payload (singular sdkKey/mobKey -// only, no sdkKeys/mobileKeys arrays) is normalized by ToParams() into a consistent accepted set. -// The wire rep's SDKKeys/MobileKeys remain nil, but params.AcceptedSDKKeys/AcceptedMobileKeys are -// synthesized from the singular fields so consumers never need to handle two code paths. -func TestEnvironmentRepOldFormatNoArrays(t *testing.T) { - jsonStr := `{ - "envID": "envid1", - "envKey": "envkey", - "envName": "envname", - "mobKey": "mob-default", - "projKey": "projkey", - "projName": "projname", - "sdkKey": { "value": "sdk-key1" }, - "secureMode": false - }` - - var rep EnvironmentRep - require.NoError(t, json.Unmarshal([]byte(jsonStr), &rep)) - assert.Nil(t, rep.SDKKeys) - assert.Nil(t, rep.MobileKeys) - - params := rep.ToParams() - assert.Equal(t, config.SDKKey("sdk-key1"), params.SDKKey) - assert.Equal(t, config.MobileKey("mob-default"), params.MobileKey) - require.Len(t, params.AcceptedSDKKeys, 1) - assert.Equal(t, AcceptedSDKKey{Value: config.SDKKey("sdk-key1")}, params.AcceptedSDKKeys[0]) - require.Len(t, params.AcceptedMobileKeys, 1) - assert.Equal(t, AcceptedMobileKey{Value: config.MobileKey("mob-default")}, params.AcceptedMobileKeys[0]) -} - -// TestNewFormatPayloadDecodesIntoOldFormatStruct pins the wire-format additive guarantee directly. The -// local oldEnvironmentRep mirrors EnvironmentRep as it was before concurrent keys: only the singular -// sdkKey (with the legacy sdkKey.expiring rotation slot) and mobKey, and none of the new sdkKeys/ -// mobileKeys arrays or per-key expiry. Decoding a full new-format payload into it must succeed and yield -// exactly the singular values an old relay binary saw before the arrays existed — because the parse path -// uses the default JSON decoder (never DisallowUnknownFields), so the unknown new fields are ignored. -func TestNewFormatPayloadDecodesIntoOldFormatStruct(t *testing.T) { - // oldEnvironmentRep is the pre-concurrent-keys shape. Do not add sdkKeys/mobileKeys/expiry here — the - // whole point is that an old binary that never knew about them still decodes a new payload cleanly. - type oldEnvironmentRep struct { - EnvID config.EnvironmentID `json:"envID"` - EnvKey string `json:"envKey"` - EnvName string `json:"envName"` - MobKey config.MobileKey `json:"mobKey"` - ProjKey string `json:"projKey"` - ProjName string `json:"projName"` - SDKKey struct { - Value config.SDKKey `json:"value"` - Expiring struct { - Value config.SDKKey `json:"value"` - Timestamp ldtime.UnixMillisecondTime `json:"timestamp"` - } `json:"expiring"` - } `json:"sdkKey"` - DefaultTTL int `json:"defaultTtl"` - SecureMode bool `json:"secureMode"` - Version int `json:"version"` - } - - // A realistic full new-format payload: singular fields plus the legacy expiring slot, the new - // sdkKeys/mobileKeys arrays, and per-key expiry — everything a current backend can emit. - jsonStr := `{ - "envID": "68e5179e8307e4099c277e2a", - "envKey": "production", - "envName": "Production", - "mobKey": "mob-f41c", - "projKey": "my-project", - "projName": "My Project", - "sdkKey": { - "value": "sdk-anchor", - "expiring": { "value": "sdk-old-anchor", "timestamp": 1699000000000 } - }, - "sdkKeys": [ - { "key": "default-sdk", "value": "sdk-anchor" }, - { "key": "service-a", "value": "sdk-service-a", "expiry": 1700000000000 } - ], - "mobileKeys": [ - { "key": "mob-key-1", "value": "mob-f41c" }, - { "key": "mob-key-2", "value": "mob-second", "expiry": 1700000000000 } - ], - "defaultTtl": 5, - "secureMode": true, - "version": 26 - }` - - var rep oldEnvironmentRep - require.NoError(t, json.Unmarshal([]byte(jsonStr), &rep)) - - // The singular fields an old relay relies on populate exactly as before; the arrays are ignored. - assert.Equal(t, config.SDKKey("sdk-anchor"), rep.SDKKey.Value) - assert.Equal(t, config.SDKKey("sdk-old-anchor"), rep.SDKKey.Expiring.Value) - assert.Equal(t, ldtime.UnixMillisecondTime(1699000000000), rep.SDKKey.Expiring.Timestamp) - assert.Equal(t, config.MobileKey("mob-f41c"), rep.MobKey) - - // The remaining scalar fields also decode unchanged. - assert.Equal(t, config.EnvironmentID("68e5179e8307e4099c277e2a"), rep.EnvID) - assert.Equal(t, "production", rep.EnvKey) - assert.Equal(t, "Production", rep.EnvName) - assert.Equal(t, "my-project", rep.ProjKey) - assert.Equal(t, "My Project", rep.ProjName) - assert.Equal(t, 5, rep.DefaultTTL) - assert.True(t, rep.SecureMode) - assert.Equal(t, 26, rep.Version) -} diff --git a/internal/envfactory/reconcile_helper_test.go b/internal/envfactory/reconcile_helper_test.go index 7ec90403..7972ff1c 100644 --- a/internal/envfactory/reconcile_helper_test.go +++ b/internal/envfactory/reconcile_helper_test.go @@ -42,24 +42,6 @@ func makeParams(sdkKey config.SDKKey, sdkKeys []AcceptedSDKKey, mobileKey config } } -// TestBuildAcceptedSet_HappyPath verifies the basic case: a single permanent SDK key that is the -// anchor, plus a mobile key and env ID. -func TestBuildAcceptedSet_HappyPath(t *testing.T) { - params := makeParams( - "sdk-anchor", - []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, - "mob-primary", - ) - set, _, err := BuildAcceptedSet(params) - - require.NoError(t, err) - expected := mustBuild(t, credential.NewAcceptedSetBuilder(). - WithEnvironmentID("env-abc"). - WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). - WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) // mobile has no identifier in makeParams fixture - assert.Equal(t, expected, set) -} - // TestBuildAcceptedSet_MultipleKeys verifies that multiple accepted SDK keys (anchor + non-anchor // permanent + expiring non-anchor) are all included in the returned AcceptedSet. func TestBuildAcceptedSet_MultipleKeys(t *testing.T) { @@ -84,127 +66,110 @@ func TestBuildAcceptedSet_MultipleKeys(t *testing.T) { assert.Equal(t, expected, set) } -// TestBuildAcceptedSet_Rename verifies that a rename — same credential value, different identifier -// — updates only the identifier in the AcceptedSet, not the accepted credential itself. The sets -// produced before and after a rename carry the same credentials but different identifier maps. -func TestBuildAcceptedSet_Rename(t *testing.T) { - paramsOldName := makeParams( - "sdk-anchor", - []AcceptedSDKKey{{Key: "old-name", Value: "sdk-anchor"}}, - "mob-primary", - ) - paramsNewName := makeParams( - "sdk-anchor", - []AcceptedSDKKey{{Key: "new-name", Value: "sdk-anchor"}}, - "mob-primary", - ) - - setOld, _, errOld := BuildAcceptedSet(paramsOldName) - setNew, _, errNew := BuildAcceptedSet(paramsNewName) - - require.NoError(t, errOld) - require.NoError(t, errNew) - // The credential content is the same — only the identifier differs. - // When Reconcile applies the new set the display name is refreshed but no credential is added or removed. - assert.NotEqual(t, setOld, setNew, "rename changes the identifier map, so the AcceptedSets differ") - expectedOld := mustBuild(t, credential.NewAcceptedSetBuilder(). - WithEnvironmentID("env-abc"). - WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("old-name")}). - WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) - assert.Equal(t, expectedOld, setOld) - expectedNew := mustBuild(t, credential.NewAcceptedSetBuilder(). - WithEnvironmentID("env-abc"). - WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("new-name")}). - WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) - assert.Equal(t, expectedNew, setNew) -} - -// TestBuildAcceptedSet_Deexpiry verifies that removing the expiry from an existing key (a -// previously expiring key that is now permanent) results in the key being permanent in the -// returned AcceptedSet. The "cancel scheduled drop" effect is realized when ReconcileCredentials -// applies this set to the Rotator. -func TestBuildAcceptedSet_Deexpiry(t *testing.T) { - // "Before" state: sdk-old has an expiry. - paramsWithExpiry := makeParams( - "sdk-anchor", - []AcceptedSDKKey{ - {Key: "default", Value: "sdk-anchor"}, - {Key: "old-key", Value: "sdk-old", Expiry: expiry1}, +// TestBuildAcceptedSet_MalformedPayloads enumerates every structurally malformed payload shape +// BuildAcceptedSet rejects. Each row guards a different silent failure, so each asserts the +// distinguishing part of its message as well as the error type: relay must reject the payload loudly +// rather than synthesize a credential-short or structurally inconsistent environment from it. +func TestBuildAcceptedSet_MalformedPayloads(t *testing.T) { + tests := []struct { + name string + params EnvironmentParams + wantMsgSubstring string + }{ + { + // A defined anchor absent from the authoritative sdkKeys[] array is structurally + // inconsistent, so it must be rejected rather than silently synthesized into the set. + name: "anchor not present in sdkKeys[]", + params: makeParams( + "sdk-anchor", + []AcceptedSDKKey{{Key: "other-key", Value: "sdk-other"}}, // anchor NOT in the array + "mob-primary", + ), + wantMsgSubstring: "not present in sdkKeys[]", }, - "mob-primary", - ) - // "After" state: sdk-old's expiry is removed — it is now permanent. - paramsNoExpiry := makeParams( - "sdk-anchor", - []AcceptedSDKKey{ - {Key: "default", Value: "sdk-anchor"}, - {Key: "old-key", Value: "sdk-old"}, // Expiry zero = permanent + { + // The mobile analogue of the anchor invariant. Without this guard the primary mobile key + // would be silently left undesignated, clearing it on reconcile and breaking event forwarding. + name: "primary mobile key not present in mobileKeys[]", + params: EnvironmentParams{ + EnvID: "env-abc", + SDKKey: "sdk-anchor", + MobileKey: "mob-primary", // defined... + AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, + AcceptedMobileKeys: []AcceptedMobileKey{ + {Key: "other", Value: "mob-other"}, // ...but NOT in the array + }, + }, + wantMsgSubstring: "not present in mobileKeys[]", }, - "mob-primary", - ) - - setWithExpiry, _, errWithExpiry := BuildAcceptedSet(paramsWithExpiry) - setNoExpiry, _, errNoExpiry := BuildAcceptedSet(paramsNoExpiry) - - require.NoError(t, errWithExpiry) - require.NoError(t, errNoExpiry) - - // The set built without expiry must include sdk-old as a permanent key. - expectedPermanent := mustBuild(t, credential.NewAcceptedSetBuilder(). - WithEnvironmentID("env-abc"). - WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). - WithSDKKey(credential.SDKKeyParams{Value: "sdk-old", Key: util.PtrOrNil("old-key")}). // permanent, no expiry - WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) - assert.Equal(t, expectedPermanent, setNoExpiry) - - // Sanity: the expiring and non-expiring versions are different. - assert.NotEqual(t, setWithExpiry, setNoExpiry) -} - -// TestBuildAcceptedSet_AnchorNotInArray verifies that a defined anchor absent from the sdkKeys[] array -// yields a *credential.MalformedCredentialSetError: the payload is structurally inconsistent (the -// designated anchor is not in the authoritative array), so it must be rejected rather than silently -// synthesized into the set. -func TestBuildAcceptedSet_AnchorNotInArray(t *testing.T) { - params := makeParams( - "sdk-anchor", - []AcceptedSDKKey{ - {Key: "other-key", Value: "sdk-other"}, // anchor NOT in the array + { + // The complement of the guard above: accepting a non-empty mobileKeys[] with no designated + // primary would clear the rotator's primary mobile key with no repoint, so event forwarding + // would keep using the previous (possibly revoked) primary instead of rejecting loudly. + name: "mobileKeys[] non-empty with no designated primary", + params: EnvironmentParams{ + EnvID: "env-abc", + SDKKey: "sdk-anchor", + MobileKey: "", // no primary designated... + AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, + AcceptedMobileKeys: []AcceptedMobileKey{ + {Key: "mob-1", Value: "mob-primary"}, // ...but the array is non-empty + }, + }, + wantMsgSubstring: "no primary mobile key is designated", }, - "mob-primary", - ) - _, _, err := BuildAcceptedSet(params) - - require.Error(t, err) - var malformed *credential.MalformedCredentialSetError - require.True(t, errors.As(err, &malformed)) - assert.Contains(t, malformed.Error(), "not present in sdkKeys[]") -} - -// TestBuildAcceptedSet_PrimaryMobileNotInArray verifies the mobile analogue of the anchor invariant: -// a defined mobKey absent from mobileKeys[] is rejected. Without this guard the primary mobile key -// would be silently left undesignated, clearing it on reconcile and breaking event forwarding. -func TestBuildAcceptedSet_PrimaryMobileNotInArray(t *testing.T) { - params := EnvironmentParams{ - EnvID: "env-abc", - SDKKey: "sdk-anchor", - MobileKey: "mob-primary", // defined... - AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, - AcceptedMobileKeys: []AcceptedMobileKey{ - {Key: "other", Value: "mob-other"}, // ...but NOT in the array + { + // An undefined anchor never matches a (defined) array value, so none is ever designated and + // Build rejects the set. + name: "anchor undefined", + params: makeParams( + "", // undefined anchor + []AcceptedSDKKey{{Key: "key-a", Value: "sdk-a"}}, + "mob-primary", + ), + wantMsgSubstring: "anchor SDK key is missing", + }, + { + // No SDK key survives at all, so the environment would have nothing to authenticate with. + name: "no usable SDK key: empty array", + params: EnvironmentParams{ + SDKKey: "", // undefined anchor + AcceptedSDKKeys: []AcceptedSDKKey{}, + AcceptedMobileKeys: []AcceptedMobileKey{}, + }, + wantMsgSubstring: "no usable SDK key in sdkKeys[]", + }, + { + // The same end state reached by filtering rather than by an empty payload. It is worth + // pinning separately: a filtered-to-empty array is a payload problem, not a caller mistake. + name: "no usable SDK key: every entry view-scoped", + params: EnvironmentParams{ + SDKKey: "", // undefined anchor + AcceptedSDKKeys: []AcceptedSDKKey{{Key: "view-sdk", Value: "sdk-viewy", HasViews: true}}, + AcceptedMobileKeys: []AcceptedMobileKey{}, + }, + wantMsgSubstring: "no usable SDK key in sdkKeys[]", }, } - _, _, err := BuildAcceptedSet(params) - require.Error(t, err) - var malformed *credential.MalformedCredentialSetError - require.True(t, errors.As(err, &malformed)) - assert.Contains(t, malformed.Error(), "not present in mobileKeys[]") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, err := BuildAcceptedSet(tt.params) + + require.Error(t, err, "a structurally malformed payload must be rejected") + var malformed *credential.MalformedCredentialSetError + require.True(t, errors.As(err, &malformed), + "every rejection from BuildAcceptedSet is a malformed-payload error") + assert.Contains(t, malformed.Error(), tt.wantMsgSubstring) + }) + } } // TestBuildAcceptedSet_NoMobileKey verifies that an environment with no mobile key (e.g. a // server-side-only environment) is valid: ToParams must not synthesize a phantom empty mobileKeys -// entry that BuildAcceptedSet would reject as malformed. +// entry that BuildAcceptedSet would reject as malformed. It also pins the boundary of the +// non-empty-mobileKeys[]-without-a-primary guard above — that guard fires only when the array is +// non-empty, and ToParams synthesizes exactly the empty array this case relies on. func TestBuildAcceptedSet_NoMobileKey(t *testing.T) { rep := EnvironmentRep{ EnvID: "env-abc", @@ -220,126 +185,6 @@ func TestBuildAcceptedSet_NoMobileKey(t *testing.T) { assert.Equal(t, expected, set) } -// TestBuildAcceptedSet_MobileKeysWithoutPrimary verifies the complement of the primary-mobile-in-array -// invariant: a non-empty mobileKeys[] with no designated primary (undefined mobKey) is rejected as a -// *credential.MalformedCredentialSetError. Without this guard the reconcile would clear the rotator's -// primary mobile key with no repoint, silently forwarding events under the previous (possibly revoked) -// primary instead of loudly rejecting the payload. -func TestBuildAcceptedSet_MobileKeysWithoutPrimary(t *testing.T) { - params := EnvironmentParams{ - EnvID: "env-abc", - SDKKey: "sdk-anchor", - MobileKey: "", // no primary designated... - AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, - AcceptedMobileKeys: []AcceptedMobileKey{ - {Key: "mob-1", Value: "mob-primary"}, // ...but the array is non-empty - }, - } - _, _, err := BuildAcceptedSet(params) - - require.Error(t, err) - var malformed *credential.MalformedCredentialSetError - require.True(t, errors.As(err, &malformed)) - assert.Contains(t, malformed.Error(), "no primary mobile key is designated") -} - -// TestBuildAcceptedSet_EmptyMobileArrayValid verifies the boundary of the guard above: an empty -// mobileKeys[] with an undefined mobKey (a server-side-only environment) is valid — the guard fires -// only when the array is non-empty, so no primary mobile key is designated and nothing is rejected. -func TestBuildAcceptedSet_EmptyMobileArrayValid(t *testing.T) { - params := EnvironmentParams{ - EnvID: "env-abc", - SDKKey: "sdk-anchor", - MobileKey: "", // undefined - AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, - AcceptedMobileKeys: []AcceptedMobileKey{}, // empty - } - set, _, err := BuildAcceptedSet(params) - - require.NoError(t, err) - expected := mustBuild(t, credential.NewAcceptedSetBuilder(). - WithEnvironmentID("env-abc"). - WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")})) - assert.Equal(t, expected, set) -} - -// TestBuildAcceptedSet_AnchorUndefined verifies that an undefined anchor (empty SDKKey) yields a -// *credential.MalformedCredentialSetError: no anchor was designated, so Build rejects the set. -func TestBuildAcceptedSet_AnchorUndefined(t *testing.T) { - params := makeParams( - "", // undefined anchor - []AcceptedSDKKey{ - {Key: "key-a", Value: "sdk-a"}, - }, - "mob-primary", - ) - _, _, err := BuildAcceptedSet(params) - - require.Error(t, err) - var malformed *credential.MalformedCredentialSetError - require.True(t, errors.As(err, &malformed)) - assert.Contains(t, malformed.Error(), "anchor SDK key is missing") -} - -// TestBuildAcceptedSet_NoSDKKeys verifies that when no SDK key survives, the payload is rejected as -// malformed. Two shapes reach this, both requiring an undefined anchor: an empty array, and an array -// whose every entry is filtered out for being scoped to a view. The second is why the case is worth -// pinning by error type — a filtered-to-empty array is a payload problem, not a caller mistake. -func TestBuildAcceptedSet_NoSDKKeys(t *testing.T) { - tests := []struct { - name string - sdkKeys []AcceptedSDKKey - }{ - {"empty array", []AcceptedSDKKey{}}, - {"every entry view-scoped", []AcceptedSDKKey{{Key: "view-sdk", Value: "sdk-viewy", HasViews: true}}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - params := EnvironmentParams{ - SDKKey: "", // undefined anchor - AcceptedSDKKeys: tt.sdkKeys, - AcceptedMobileKeys: []AcceptedMobileKey{}, - } - _, _, err := BuildAcceptedSet(params) - - require.Error(t, err, "a set with no SDK key at all must be rejected") - var malformed *credential.MalformedCredentialSetError - require.ErrorAs(t, err, &malformed, "every rejection from BuildAcceptedSet is a malformed-payload error") - }) - } -} - -// TestBuildAcceptedSet_MixedUpdate verifies add + re-anchor + remove in a single params update -// produces an AcceptedSet that contains the right keys in the right state. The ordering -// (add → re-anchor → remove) is enforced by ReconcileCredentials when it consumes this set; -// this test only asserts the AcceptedSet content. -func TestBuildAcceptedSet_MixedUpdate(t *testing.T) { - // New state after the patch: - // - sdk-new-anchor is the new anchor (re-anchor) - // - sdk-b carries over unchanged - // - sdk-c is newly added - // - sdk-old-anchor is gone (remove) - params := makeParams( - "sdk-new-anchor", - []AcceptedSDKKey{ - {Key: "new-default", Value: "sdk-new-anchor"}, // re-anchor - {Key: "service-b", Value: "sdk-b"}, // unchanged - {Key: "service-c", Value: "sdk-c"}, // added - }, - "mob-primary", - ) - set, _, err := BuildAcceptedSet(params) - - require.NoError(t, err) - expected := mustBuild(t, credential.NewAcceptedSetBuilder(). - WithEnvironmentID("env-abc"). - WithAnchor(credential.SDKKeyParams{Value: "sdk-new-anchor", Key: util.PtrOrNil("new-default")}). - WithSDKKey(credential.SDKKeyParams{Value: "sdk-b", Key: util.PtrOrNil("service-b")}). - WithSDKKey(credential.SDKKeyParams{Value: "sdk-c", Key: util.PtrOrNil("service-c")}). - WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary"})) - assert.Equal(t, expected, set) -} - // TestBuildAcceptedSet_AnchorNeverExpiring verifies the invariant defense: even if a payload // carries an expiry on the anchor's own entry, the anchor is added as a permanent key, never // as an expiring one. @@ -364,35 +209,11 @@ func TestBuildAcceptedSet_AnchorNeverExpiring(t *testing.T) { assert.Equal(t, expected, set) } -// TestBuildAcceptedSet_MultipleMobileKeys verifies that all accepted mobile keys are included, -// exercising the len(AcceptedMobileKeys) > 1 path, and that the wire's mobKey is designated as the -// primary mobile key. -func TestBuildAcceptedSet_MultipleMobileKeys(t *testing.T) { - params := EnvironmentParams{ - EnvID: "env-abc", - SDKKey: "sdk-anchor", - MobileKey: "mob-primary", - AcceptedSDKKeys: []AcceptedSDKKey{{Key: "default", Value: "sdk-anchor"}}, - AcceptedMobileKeys: []AcceptedMobileKey{ - {Key: "mob-1", Value: "mob-primary"}, - {Key: "mob-2", Value: "mob-secondary"}, - }, - } - set, _, err := BuildAcceptedSet(params) - - require.NoError(t, err) - expected := mustBuild(t, credential.NewAcceptedSetBuilder(). - WithEnvironmentID("env-abc"). - WithAnchor(credential.SDKKeyParams{Value: "sdk-anchor", Key: util.PtrOrNil("default")}). - WithMobileKey(credential.MobileKeyParams{Value: "mob-secondary", Key: util.PtrOrNil("mob-2")}). - WithPrimaryMobileKey(credential.MobileKeyParams{Value: "mob-primary", Key: util.PtrOrNil("mob-1")})) - assert.Equal(t, expected, set) -} - // TestBuildAcceptedSet_ExpiringMobileKey verifies that a mobile key carrying a non-zero Expiry is // plumbed through as an expiring key (parallel to the expiring-SDK-key path), while the permanent // primary mobile key is designated. This is what makes per-key mobile expiry work end-to-end: -// params carry it → BuildAcceptedSet plumbs it into the AcceptedSet → Reconcile acts on it. +// params carry it → BuildAcceptedSet plumbs it into the AcceptedSet → Reconcile acts on it. It also +// exercises the len(AcceptedMobileKeys) > 1 path and the designation of the wire's mobKey as primary. func TestBuildAcceptedSet_ExpiringMobileKey(t *testing.T) { params := EnvironmentParams{ EnvID: "env-abc", @@ -455,13 +276,6 @@ func TestBuildAcceptedSet_ViewScopedKeys(t *testing.T) { wantSet func() *credential.AcceptedSetBuilder wantRejected []string }{ - { - // Baseline: nothing view-scoped behaves exactly as it did before the field existed. - name: "no view-scoped keys", - sdkKeys: []AcceptedSDKKey{anchorEntry, extraSDK}, - mobileKeys: []AcceptedMobileKey{primaryEntry, extraMob}, - wantSet: baseWithExtras, - }, { name: "view-scoped non-anchor SDK key is excluded", sdkKeys: []AcceptedSDKKey{anchorEntry, extraSDK, {Key: "view-sdk", Value: "sdk-viewy", HasViews: true}}, diff --git a/internal/events/event_publisher_test.go b/internal/events/event_publisher_test.go index 6a67ce76..1881f39a 100644 --- a/internal/events/event_publisher_test.go +++ b/internal/events/event_publisher_test.go @@ -189,18 +189,6 @@ func TestHTTPEventPublisherCapacity(t *testing.T) { }) } -func TestInitialQueueCapacity(t *testing.T) { - // Unset initial capacity preallocates the full capacity -- the original behavior, used by the - // analytics publisher, which never sets OptionInitialCapacity. - assert.Equal(t, 1000, initialQueueCapacity(1000, 0)) - assert.Equal(t, 10000, initialQueueCapacity(10000, 0)) - // A smaller initial capacity is used as-is, letting the queue start small and grow. - assert.Equal(t, 1000, initialQueueCapacity(10000, 1000)) - // The initial allocation is never larger than the maximum capacity. - assert.Equal(t, 1000, initialQueueCapacity(1000, 1000)) - assert.Equal(t, 1000, initialQueueCapacity(1000, 5000)) -} - func TestHTTPEventPublisherInitialCapacityGrowsToCapacity(t *testing.T) { // With an initial capacity smaller than the (maximum) capacity, the queue must still grow past // the initial allocation and only drop events once the maximum capacity is reached. diff --git a/internal/relayenv/env_context_credential_serialization_test.go b/internal/relayenv/env_context_credential_serialization_test.go deleted file mode 100644 index 76931657..00000000 --- a/internal/relayenv/env_context_credential_serialization_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package relayenv - -// Regression for the deferred-flip concurrency hazard: the cleanup ticker's triggerCredentialChanges -// must be serialized against reconcileCredentials via reconcileMu. Because Reconcile queues the new -// anchor's addition but defers the pointer flip to CommitAnchor, a ticker that drained that addition in -// the window between them would run addCredential with the anchor still on the old key, skip the new -// anchor's startSDKClient, and leave the env with no upstream client. reconcileMu closes that window. - -import ( - "testing" - "time" - - st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" - "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" - - "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" - - "github.com/stretchr/testify/require" -) - -func TestCredentialTickerIsSerializedAgainstReconcile(t *testing.T) { - envConfig := st.EnvMain.Config - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - clientCh := make(chan *testclient.FakeLDClient, 10) - readyCh := make(chan EnvContext, 1) - env := makeBasicEnv(t, envConfig, testclient.FakeLDClientFactoryWithChannel(true, clientCh), mockLog.Loggers, readyCh) - defer env.Close() - - require.Equal(t, env, requireEnvReady(t, readyCh)) - requireClientReady(t, clientCh) - - envImpl := env.(*envContextImpl) - - // Stand in for an in-flight reconcileCredentials by holding reconcileMu: while it is held, the - // cleanup ticker's triggerCredentialChanges must NOT run (that is exactly the interleaving that would - // steal a queued addition mid-re-anchor). - envImpl.reconcileMu.Lock() - - tickerDone := make(chan struct{}) - go func() { - envImpl.triggerCredentialChanges(time.Unix(3000, 0)) - close(tickerDone) - }() - - select { - case <-tickerDone: - envImpl.reconcileMu.Unlock() - t.Fatal("triggerCredentialChanges ran while reconcileMu was held: the ticker is not serialized against reconcile") - case <-time.After(100 * time.Millisecond): - // Expected: the ticker is blocked on reconcileMu. - } - - // Once the "reconcile" releases the lock, the ticker proceeds. - envImpl.reconcileMu.Unlock() - select { - case <-tickerDone: - case <-time.After(time.Second): - t.Fatal("triggerCredentialChanges did not proceed after reconcileMu was released") - } -} diff --git a/internal/relayenv/env_context_impl_test.go b/internal/relayenv/env_context_impl_test.go index cf7eab5b..19081781 100644 --- a/internal/relayenv/env_context_impl_test.go +++ b/internal/relayenv/env_context_impl_test.go @@ -406,6 +406,12 @@ func TestMobileKeyReconcileExpiry(t *testing.T) { assert.Contains(t, env.GetCredentials(), primaryMobile) } +// TestNonAnchorSDKKeysDoNotOpenUpstreamClient verifies that non-anchor SDK keys are accepted without +// opening an upstream client of their own: they share the anchor's single upstream connection. +// +// It also pins the GetClient contract that follows from it. Non-anchor keys have no client, so GetClient +// must keep returning the anchor's — never nil, never a non-anchor client. That is what callers depend +// on: nil means "env not ready"; non-nil means "use this client." func TestNonAnchorSDKKeysDoNotOpenUpstreamClient(t *testing.T) { envConfig := st.EnvMain.Config readyCh := make(chan EnvContext, 1) @@ -424,6 +430,9 @@ func TestNonAnchorSDKKeysDoNotOpenUpstreamClient(t *testing.T) { anchorClient := requireClientReady(t, clientCh) assert.Equal(t, envConfig.SDKKey, anchorClient.Key) + // GetClient returns the anchor's client even before any non-anchor keys are added. + assert.Equal(t, anchorClient, env.GetClient()) + nonAnchorKey1 := config.SDKKey("non-anchor-key-1") nonAnchorKey2 := config.SDKKey("non-anchor-key-2") @@ -446,45 +455,6 @@ func TestNonAnchorSDKKeysDoNotOpenUpstreamClient(t *testing.T) { if !helpers.AssertNoMoreValues(t, clientCh, 200*time.Millisecond) { t.FailNow() } -} - -// TestGetClientReturnsAnchorInMultiKeyEnv verifies that GetClient returns the anchor's upstream -// client when the environment holds multiple SDK keys. Non-anchor SDK keys share the same -// upstream connection (the anchor's), so GetClient must never return a non-anchor client -// and must remain non-nil after non-anchor keys are added. This is the contract callers of -// GetClient depend on: nil means "env not ready"; non-nil means "use this client." -func TestGetClientReturnsAnchorInMultiKeyEnv(t *testing.T) { - envConfig := st.EnvMain.Config - readyCh := make(chan EnvContext, 1) - clientCh := make(chan *testclient.FakeLDClient, 10) - clientFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - env := makeBasicEnv(t, envConfig, clientFactory, mockLog.Loggers, readyCh) - defer env.Close() - - assert.Equal(t, env, requireEnvReady(t, readyCh)) - anchorClient := requireClientReady(t, clientCh) - assert.Equal(t, envConfig.SDKKey, anchorClient.Key) - - // GetClient must return the anchor's client even before any non-anchor keys are added. - assert.Equal(t, anchorClient, env.GetClient()) - - nonAnchorKey1 := config.SDKKey("non-anchor-key-1") - nonAnchorKey2 := config.SDKKey("non-anchor-key-2") - - env.ReconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithAnchor(credential.SDKKeyParams{Value: envConfig.SDKKey}). - WithSDKKey(credential.SDKKeyParams{Value: nonAnchorKey1}). - WithSDKKey(credential.SDKKeyParams{Value: nonAnchorKey2}))) - - // No new upstream client was created for the non-anchor keys. - if !helpers.AssertNoMoreValues(t, clientCh, 200*time.Millisecond) { - t.FailNow() - } // GetClient still returns the anchor's client — not nil, not a non-anchor client. assert.Equal(t, anchorClient, env.GetClient()) @@ -547,72 +517,6 @@ func TestNonPrimaryMobileKeyDoesNotStealEventForwarding(t *testing.T) { }) } -// When an SDK key that is still accepted in its grace period is re-anchored back into the primary -// slot, a fresh SDK client is built for it (its previous client was closed when its demotion -// committed -- the anchor owns the env's single upstream connection, so a demoted key keeps only its -// credential mappings). Originally a regression test from #716 for the old UpdateCredential path, -// where re-anchoring to a key still in its grace period spawned a fresh client and orphaned the old -// one. Under the ReconcileCredentials model that leak is structurally impossible: every displaced -// anchor's client is closed at commit, so no rotation sequence can leave two live upstream clients. -func TestReAnchoringToKeyStillInGraceBuildsFreshClient(t *testing.T) { - envConfig := st.EnvMain.Config - keyA := envConfig.SDKKey - keyB := config.SDKKey("keyB") - readyCh := make(chan EnvContext, 1) - - clientCh := make(chan *testclient.FakeLDClient, 10) - clientFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - env := makeBasicEnv(t, envConfig, clientFactory, mockLog.Loggers, readyCh) - defer env.Close() - - assert.Equal(t, env, requireEnvReady(t, readyCh)) - clientA1 := requireClientReady(t, clientCh) - assert.Equal(t, env.GetClient(), clientA1) - - start := time.Unix(1000, 0) - - // Rotate keyA -> keyB, deprecating keyA with an hour-long grace. keyA stays accepted during the - // grace window, but its client (clientA1) is closed as soon as the re-anchor commits. - env.(*envContextImpl).reconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder(). - WithAnchor(credential.SDKKeyParams{Value: keyB}). - WithSDKKey(credential.SDKKeyParams{Value: keyA, Expiry: util.PtrOrNil(start.Add(1 * time.Hour))})), - start) - - clientB := requireClientReady(t, clientCh) - assert.NotEqual(t, clientA1, clientB) - if !helpers.AssertChannelClosed(t, clientA1.CloseCh, time.Second, "clientA1 should have been closed when keyA was demoted") { - t.FailNow() - } - - // Re-anchor back to keyA while it is still within its grace period. keyA's credential mappings - // survived the demotion but its client did not, so a fresh client is built for it. keyB is omitted - // from the set (no expiry), so it is revoked immediately; its client closes at commit. - env.(*envContextImpl).reconcileCredentials( - mustBuildAcceptedSet(t, credential.NewAcceptedSetBuilder().WithAnchor(credential.SDKKeyParams{Value: keyA})), - start.Add(10*time.Minute)) - - // A fresh client was built for keyA -- the demotion closed its original one. - clientA2 := requireClientReady(t, clientCh) - assert.NotEqual(t, clientA1, clientA2) - // keyB was displaced by the re-anchor, so its client is closed. - if !helpers.AssertChannelClosed(t, clientB.CloseCh, time.Second, "client for the displaced keyB should have been closed") { - t.FailNow() - } - - require.Eventually(t, func() bool { - return env.GetClient() == clientA2 - }, time.Second, 10*time.Millisecond, "env.GetClient() should return the fresh client for keyA after re-anchor") - - creds := env.GetCredentials() - assert.Contains(t, creds, keyA) - assert.NotContains(t, creds, keyB) -} - // gatedClientFactory wraps the normal fake factory but blocks the factory call for gateKey until // `gate` is closed, signalling on `started` once that call is in flight. This lets a test interleave // a credential revocation with an in-flight startSDKClient that has not yet taken c.mu. diff --git a/internal/relayenv/env_context_reanchor_behavior_test.go b/internal/relayenv/env_context_reanchor_behavior_test.go index a99a66ca..c2daf797 100644 --- a/internal/relayenv/env_context_reanchor_behavior_test.go +++ b/internal/relayenv/env_context_reanchor_behavior_test.go @@ -1,18 +1,15 @@ package relayenv -// Permanent behavioral regression tests for re-anchoring. These pin three properties of the +// Permanent behavioral regression tests for re-anchoring. These pin two properties of the // upstream-client swap: // // - an open downstream (client-side) connection survives a re-anchor and keeps receiving events, // with the re-wired big-segment synchronizer driving its invalidations; -// - the new anchor's initial sync re-broadcasts a full "put" downstream, so a downstream SDK sees one -// duplicate put per re-anchor (tolerable — SDKs apply puts idempotently — but the swap must expect it); // - httpconfig carries no baked-in SDK key other than the Authorization header the SDK sets per client, // so it needs no re-wiring on a re-anchor. import ( "net/http" - "sync" "testing" "time" @@ -22,7 +19,6 @@ import ( "github.com/launchdarkly/ld-relay/v8/internal/httpconfig" st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" - "github.com/launchdarkly/ld-relay/v8/internal/store" "github.com/launchdarkly/ld-relay/v8/internal/streams" "github.com/launchdarkly/eventsource" @@ -30,37 +26,12 @@ import ( "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" "github.com/launchdarkly/go-server-sdk/v7/ldcomponents" "github.com/launchdarkly/go-server-sdk/v7/subsystems" - "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoretypes" helpers "github.com/launchdarkly/go-test-helpers/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// putCountingStreamUpdates is a streams.EnvStreamUpdates that counts the "all data" broadcasts it -// receives, so a test can observe how many full "put"s a sequence of store inits produces downstream. -type putCountingStreamUpdates struct { - mu sync.Mutex - allDataUpdates int -} - -func (r *putCountingStreamUpdates) SendAllDataUpdate(_ []ldstoretypes.Collection) { - r.mu.Lock() - r.allDataUpdates++ - r.mu.Unlock() -} - -func (r *putCountingStreamUpdates) SendSingleItemUpdate(_ ldstoretypes.DataKind, _ string, _ ldstoretypes.ItemDescriptor) { -} - -func (r *putCountingStreamUpdates) InvalidateClientSideState() {} - -func (r *putCountingStreamUpdates) allDataCount() int { - r.mu.Lock() - defer r.mu.Unlock() - return r.allDataUpdates -} - // TestReanchorDownstreamConnectionSurvives verifies that an open downstream client-side connection // survives a re-anchor and keeps receiving events. The connection is keyed on the environment ID (a // scoped credential) and is independent of the upstream SDK anchor key, so swapping the anchor must not @@ -129,29 +100,6 @@ func TestReanchorDownstreamConnectionSurvives(t *testing.T) { }) } -// TestReanchorInitialSyncRebroadcastsPut verifies that the new anchor's client performs its own initial -// sync when it comes up, re-broadcasting a full "put" to every connected downstream stream. From a -// downstream SDK's perspective this is a duplicate put on each re-anchor. It is tolerable — SDKs apply -// puts idempotently — but the re-anchor implementation must expect it; it is not a corruption. -func TestReanchorInitialSyncRebroadcastsPut(t *testing.T) { - rec := &putCountingStreamUpdates{} - adapter := store.NewSSERelayDataStoreAdapter(ldcomponents.InMemoryDataStore(), rec) - - // The original anchor client builds and performs its initial sync -> one downstream "put". - s1, err := adapter.Build(subsystems.BasicClientContext{}) - require.NoError(t, err) - require.NoError(t, s1.Init(st.AllData)) - require.Equal(t, 1, rec.allDataCount()) - - // Re-anchor: the new anchor's client performs its OWN initial sync (store handover hands it the same - // wrapper, but the new client still re-broadcasts a full put when it initializes). - s2, err := adapter.Build(subsystems.BasicClientContext{}) - require.NoError(t, err) - require.NoError(t, s2.Init(st.AllData)) - - assert.Equal(t, 2, rec.allDataCount(), "the new anchor's initial sync re-broadcasts a full put") -} - // TestReanchorHTTPConfigIsKeyIndependent verifies that httpconfig carries no baked-in SDK key other than // the Authorization default header, so a re-anchor needs no httpconfig re-wire. Relay injects the SDK // HTTP config *builder* into the SDK config, and the SDK rebuilds the HTTP config with the new anchor key diff --git a/internal/relayenv/env_context_reanchor_bigsegment_test.go b/internal/relayenv/env_context_reanchor_bigsegment_test.go index 43437943..83394f55 100644 --- a/internal/relayenv/env_context_reanchor_bigsegment_test.go +++ b/internal/relayenv/env_context_reanchor_bigsegment_test.go @@ -1,8 +1,12 @@ package relayenv // Tests that the big-segment synchronizer follows the anchor across a re-anchor: recreated on the -// new anchor key, an already-started sync continues while the old one closes, a rolled-back -// re-anchor does not rewire, and a not-configured environment is a no-op. +// new anchor key, an already-started sync continues while the old one closes, and a rolled-back +// re-anchor does not rewire. +// +// The unconfigured case (bigSegmentSync nil) needs no test of its own: makeBasicEnv leaves the field +// nil, so every re-anchor test built on it would panic in reanchorBigSegmentSync's old.Close() if the +// nil guard were removed. import ( "testing" @@ -126,32 +130,6 @@ func TestReanchorBigSegmentSync_RollbackDoesNotRewire(t *testing.T) { assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "anchor unchanged after rollback") } -// TestReanchorBigSegmentSync_NotConfiguredIsNoOp: when big segments are not configured there is no -// synchronizer, and a re-anchor must be a no-op for big-segment sync (no creation, no panic). -func TestReanchorBigSegmentSync_NotConfiguredIsNoOp(t *testing.T) { - envConfig := st.EnvMain.Config - capturing := &capturingBigSegmentSynchronizerFactory{} - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - clientCh := make(chan *testclient.FakeLDClient, 10) - noStore := func(config.EnvConfig, config.Config, ldlog.Loggers) (bigsegments.BigSegmentStore, error) { - return nil, nil // no big-segment store -> no synchronizer - } - env := newBigSegmentTestEnv(t, noStore, - testclient.FakeLDClientFactoryWithChannel(true, clientCh), capturing, mockLog.Loggers) - defer env.Close() - - reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) - - count, _ := capturing.snapshot() - assert.Equal(t, 0, count, "no synchronizer is created when big segments are not configured") - assert.Equal(t, reanchorTestKey2, env.(*envContextImpl).keyRotator.AnchorKey(), "the SDK re-anchor still committed") -} - -// reanchorTestKey3 is a third anchor SDK key, used to drive A->B->C sequential re-anchors. -const reanchorTestKey3 = config.SDKKey("reanchor-new-anchor-3") - // TestReanchorBigSegmentSync_ReanchorBeforeFirstSegmentThenStartsNewSync covers the ordering where a // re-anchor happens BEFORE any big segment has appeared (so the replacement is built but not started), // and then the first segment appears. setBigSegmentsExist must start the CURRENT (new) synchronizer, @@ -184,45 +162,6 @@ func TestReanchorBigSegmentSync_ReanchorBeforeFirstSegmentThenStartsNewSync(t *t assert.False(t, oldSync.isStarted(), "the retired synchronizer is never started") } -// TestReanchorBigSegmentSync_MultipleSequentialReanchors drives A->B->C and asserts each intermediate -// synchronizer is Closed, only the final one is current+Started, and the create bookkeeping tracks each -// anchor key. Guards against synchronizer/consumer accumulation and stale-key bugs across rotations. -func TestReanchorBigSegmentSync_MultipleSequentialReanchors(t *testing.T) { - envConfig := st.EnvMain.Config - capturing := &capturingBigSegmentSynchronizerFactory{} - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - clientCh := make(chan *testclient.FakeLDClient, 10) - env := newBigSegmentTestEnv(t, nullBigSegmentStoreFactory, - testclient.FakeLDClientFactoryWithChannel(true, clientCh), capturing, mockLog.Loggers) - defer env.Close() - envImpl := env.(*envContextImpl) - - envImpl.setBigSegmentsExist() - syncA := capturing.latest() - require.True(t, syncA.isStarted()) - - // A -> B. - reanchor(t, env, reanchorTestKey2, envConfig.SDKKey, time.Unix(1000, 0)) - syncB := capturing.latest() - require.NotSame(t, syncA, syncB) - assert.True(t, syncA.isClosed(), "A is closed after A->B") - assert.True(t, syncB.isStarted(), "B is started (a segment already existed)") - - // B -> C. - reanchor(t, env, reanchorTestKey3, reanchorTestKey2, time.Unix(1000, 0)) - syncC := capturing.latest() - require.NotSame(t, syncB, syncC) - assert.True(t, syncB.isClosed(), "B is closed after B->C") - assert.True(t, syncC.isStarted(), "C is started") - - count, sdkKey := capturing.snapshot() - assert.Equal(t, 3, count, "one synchronizer per anchor: A, B, C") - assert.Equal(t, reanchorTestKey3, sdkKey, "the current synchronizer is on the final anchor key") - assert.Equal(t, reanchorTestKey3, envImpl.keyRotator.AnchorKey(), "the SDK anchor is C") -} - // TestReanchorBigSegmentSync_ConcurrentStoreUpdateDuringReanchorIsRaceFree is a regression test for the // data race introduced when re-anchor made c.bigSegmentSync runtime-mutable: the store-update sink reads // that field to decide whether to check for big segments, on the SDK data-source goroutine, while a @@ -275,6 +214,11 @@ func TestReanchorBigSegmentSync_ConcurrentStoreUpdateDuringReanchorIsRaceFree(t // synchronizer bound to A is created and Started (a segment already exists), and B's synchronizer is // Closed. This pins that the "previously-accepted key" promotion path does not shortcut the big-segment // re-wire. +// +// It is also the sequential-re-anchor case: three commits produce three synchronizers with every +// intermediate Closed and only the final one current and Started, so synchronizers and their consumer +// goroutines cannot accumulate across rotations. reanchorBigSegmentSync does not branch on whether the +// third key is brand new or a re-promotion, so this covers A->B->C as well. func TestReanchorBigSegmentSync_RepromoteInGraceFormerAnchorRewires(t *testing.T) { envConfig := st.EnvMain.Config capturing := &capturingBigSegmentSynchronizerFactory{} diff --git a/internal/relayenv/env_context_reanchor_helpers_test.go b/internal/relayenv/env_context_reanchor_helpers_test.go index e3e0f3df..7a6ab22d 100644 --- a/internal/relayenv/env_context_reanchor_helpers_test.go +++ b/internal/relayenv/env_context_reanchor_helpers_test.go @@ -54,12 +54,6 @@ func (r *recordingStreamUpdates) InvalidateClientSideState() { r.mu.Unlock() } -func (r *recordingStreamUpdates) allDataCount() int { - r.mu.Lock() - defer r.mu.Unlock() - return r.allDataUpdates -} - // reanchor re-anchors env onto newKey while keeping oldKey accepted for a grace hour (the old // client stays up while the new one is built, then closes when the commit lands). This mirrors the // backend's default-rotation behavior: the new anchor is non-expiring, the demoted old anchor diff --git a/internal/relayenv/env_context_reanchor_rollback_test.go b/internal/relayenv/env_context_reanchor_rollback_test.go deleted file mode 100644 index 6337c3d3..00000000 --- a/internal/relayenv/env_context_reanchor_rollback_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package relayenv - -// Regression: a grace-demotion re-anchor rollback must not leave the still-authoritative anchor exposed -// to the cleanup ticker. -// -// Scenario (the default backend rotation): anchor A is demoted with a +1h grace expiry while brand-new -// key B is designated the new anchor. B's client fails to initialize, so the synchronous re-anchor rolls -// back: A stays the anchor and keeps serving. A's accepted entry still carries the grace expiry and -// CommitAnchor never ran (A is still anchorKey), so the cleanup ticker firing past the grace window must -// NOT expire A and close the env's only client -- StepTime's anchor guard prevents that. Without the -// guard, GetClient() returns nil while the rotator still names A the anchor: a silent upstream outage. - -import ( - "errors" - "testing" - "time" - - "github.com/launchdarkly/ld-relay/v8/config" - "github.com/launchdarkly/ld-relay/v8/internal/sdks" - st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" - "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" - - "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" - ld "github.com/launchdarkly/go-server-sdk/v7" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReanchorRollbackGraceDemotionKeepsAnchorServing(t *testing.T) { - envConfig := st.EnvMain.Config - fakeErr := errors.New("re-anchor: new client init refused") - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - clientCh := make(chan *testclient.FakeLDClient, 10) - healthyFactory := testclient.FakeLDClientFactoryWithChannel(true, clientCh) - // Fail only for the new anchor; the original anchor builds fine. - factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { - if sdkKey == reanchorSyncTestKey2 { - return nil, fakeErr - } - return healthyFactory(sdkKey, cfg, timeout) - } - - readyCh := make(chan EnvContext, 1) - env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) - defer env.Close() - - require.Equal(t, env, requireEnvReady(t, readyCh)) - originalClient := requireClientReady(t, clientCh) - require.Eventually(t, func() bool { return env.GetClient() == originalClient }, time.Second, 10*time.Millisecond) - - now := time.Unix(2000, 0) - // Re-anchor A->B with A grace-demoted (+1h). B's build fails, so this rolls back. - reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) - - // Rollback holds immediately: A is still the anchor and still serving. - require.Same(t, originalClient, env.GetClient(), "rollback keeps the old anchor serving") - require.Equal(t, envConfig.SDKKey, env.(*envContextImpl).keyRotator.AnchorKey(), "anchor stayed on the old key") - - // The cleanup ticker fires after the grace window. A is STILL the anchor, so it must keep serving -- - // a key's grace expiry must not apply to it while it is the authoritative anchor. - env.(*envContextImpl).triggerCredentialChanges(now.Add(time.Hour + time.Minute)) - - assert.Equal(t, envConfig.SDKKey, env.(*envContextImpl).keyRotator.AnchorKey(), - "the anchor pointer is unchanged after the ticker") - assert.NotNil(t, env.GetClient(), - "the cleanup ticker must not reap the still-authoritative anchor's client (env would go dark)") -} diff --git a/internal/relayenv/env_context_reanchor_synchronous_test.go b/internal/relayenv/env_context_reanchor_synchronous_test.go index 9483100d..18d5f381 100644 --- a/internal/relayenv/env_context_reanchor_synchronous_test.go +++ b/internal/relayenv/env_context_reanchor_synchronous_test.go @@ -179,6 +179,11 @@ func TestReanchorSync_CaseA_InitFailureRollsBack(t *testing.T) { // re-anchor B→A while A is still in its grace period. A's credential mappings survived the // demotion, but its client did not, so the second re-anchor must build a fresh client for A and // close B's client at commit. +// +// The second re-anchor omits B from the payload entirely rather than grace-demoting it, so it also +// pins the immediate-revocation shape: a displaced anchor that the payload drops outright leaves the +// accepted set as well as losing its client. (The grace-demotion shape is the first re-anchor here, +// and TestReanchorSync_CaseA_BuildsNewClientAndMovesAnchor asserts the demoted key stays accepted.) func TestReanchorSync_CaseB_RepromoteInGraceKeyBuildsFreshClient(t *testing.T) { envConfig := st.EnvMain.Config @@ -209,15 +214,18 @@ func TestReanchorSync_CaseB_RepromoteInGraceKeyBuildsFreshClient(t *testing.T) { envImpl.mu.RUnlock() require.False(t, originalStillPresent, "demoted original anchor's client removed at commit") - // Second re-anchor: key2 → original. The original key is still accepted (its mappings were never - // torn down) but it has no client, so a fresh one is built; key2's client closes at commit. - reanchorViaReconcile(t, env, envConfig.SDKKey, reanchorSyncTestKey2, "", envConfig.MobileKey, envConfig.EnvID, now) + // Second re-anchor: key2 → original, with key2 omitted from the payload entirely (revoked outright, + // not grace-demoted). The original key is still accepted (its mappings were never torn down) but it + // has no client, so a fresh one is built; key2's client closes at commit. + reanchorViaReconcile(t, env, envConfig.SDKKey, "", "", envConfig.MobileKey, envConfig.EnvID, now) freshClient := requireClientReady(t, clientCh) assert.NotSame(t, originalClient, freshClient, "re-promoting an in-grace key builds a fresh client") assert.Same(t, freshClient, env.GetClient(), "the fresh client is current after the re-anchor") assert.Equal(t, envConfig.SDKKey, envImpl.keyRotator.AnchorKey(), "anchor flipped back to the original key") key2Client.AwaitClose(t, time.Second) + assert.NotContains(t, env.GetCredentials(), credential.SDKCredential(reanchorSyncTestKey2), + "a displaced anchor omitted from the payload is revoked outright, not left accepted in grace") } // TestReanchorSync_CredentialExpiryDuringReanchorIsSerialized exercises the concurrency gap closed @@ -480,9 +488,6 @@ func TestReanchorSync_Offline_CommitsWithoutBuildingClient(t *testing.T) { require.Eventually(t, func() bool { return env.GetClient() == initialClient }, time.Second, 10*time.Millisecond) envImpl := env.(*envContextImpl) - envImpl.mu.RLock() - genBefore := envImpl.anchorClientGen - envImpl.mu.RUnlock() now := time.Unix(2000, 0) reanchorViaReconcile(t, env, reanchorSyncTestKey2, envConfig.SDKKey, "", envConfig.MobileKey, envConfig.EnvID, now) @@ -496,15 +501,6 @@ func TestReanchorSync_Offline_CommitsWithoutBuildingClient(t *testing.T) { } assert.NoError(t, env.GetInitError()) - // The generation guard exists to protect a replacement client's install from a stale, still-in-flight - // build. An offline commit installs no replacement, so bumping it protects nothing -- it would only - // strand a build launched before this commit (e.g. the initial client at construction, generation 0) - // by making it see itself as superseded when it later finishes. Offline commits leave it untouched. - envImpl.mu.RLock() - genAfter := envImpl.anchorClientGen - envImpl.mu.RUnlock() - assert.Equal(t, genBefore, genAfter, "an offline re-anchor commit must not advance anchorClientGen") - // The single offline client survives the rotation: it is not closed and GetClient still finds it. if !helpers.AssertChannelNotClosed(t, initialClient.CloseCh, 100*time.Millisecond, "the offline env's only client must not be closed by a re-anchor") { @@ -514,7 +510,7 @@ func TestReanchorSync_Offline_CommitsWithoutBuildingClient(t *testing.T) { } // TestReanchorSync_Offline_ReanchorDuringInitialBuildDoesNotStrandClient drives the failure scenario -// the anchorClientGen guard above prevents: an offline re-anchor commits while the environment's +// the anchorClientGen guard prevents: an offline re-anchor commits while the environment's // initial client build (launched at construction with generation 0) is still in flight. Before the // fix, that commit's unconditional generation bump made the in-flight build see itself as superseded // once it finished, so it discarded itself -- and because an offline re-anchor never builds a @@ -689,69 +685,6 @@ func TestReanchorSync_PreviouslyAcceptedAnchorPromotionFailureKeepsItsMappings(t assert.NoError(t, env.GetInitError()) } -// TestReanchor_SupersededFailingBuildDoesNotClobberInitErr: the initial startSDKClient(A) is still in -// flight when a re-anchor to healthy B commits (initErr=nil). A's build then fails -- returning a -// NON-NIL, uninitialized client with the error, exactly as the real SDK's MakeCustomClient does. Because -// a re-anchor committed since this build launched, it is superseded: it must be discarded, not installed, -// and must not touch initErr -- otherwise it would clobber a healthy env into a whole-env 401. -func TestReanchor_SupersededFailingBuildDoesNotClobberInitErr(t *testing.T) { - envConfig := st.EnvMain.Config - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - clientCh := make(chan *testclient.FakeLDClient, 10) - healthy := testclient.FakeLDClientFactoryWithChannel(true, clientCh) - - gate := make(chan struct{}) - entered := make(chan struct{}, 1) - factory := func(sdkKey config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { - if sdkKey == envConfig.SDKKey { - // The ORIGINAL anchor A: block, then fail like the real SDK -- a non-nil, uninitialized client - // plus the error (MakeCustomClient returns the client on init failure/timeout). - entered <- struct{}{} - <-gate - return &testclient.FakeLDClient{Key: sdkKey, CloseCh: make(chan struct{})}, ld.ErrInitializationFailed - } - return healthy(sdkKey, cfg, timeout) - } - - readyCh := make(chan EnvContext, 1) - env := makeBasicEnv(t, envConfig, factory, mockLog.Loggers, readyCh) - defer env.Close() - - envImpl := env.(*envContextImpl) - <-entered // A's initial build is blocked; anchor is still A. - - // Re-anchor A -> B (B brand new/healthy; A grace-demoted +1h). B builds, commits, clears initErr. - now := time.Unix(2000, 0) - expiry := now.Add(time.Hour) - set, err := credential.NewAcceptedSetBuilder(). - WithAnchor(credential.SDKKeyParams{Value: reanchorSyncTestKey2}). - WithSDKKey(credential.SDKKeyParams{Value: envConfig.SDKKey, Expiry: &expiry}). - WithPrimaryMobileKey(credential.MobileKeyParams{Value: envConfig.MobileKey}). - WithEnvironmentID(envConfig.EnvID). - Build() - require.NoError(t, err) - envImpl.reconcileCredentials(set, now) - require.Equal(t, reanchorSyncTestKey2, envImpl.keyRotator.AnchorKey()) - require.NoError(t, env.GetInitError(), "sanity: healthy after re-anchor to B") - - // The stale initial A build now fails late. - close(gate) - <-readyCh - - assert.NoError(t, env.GetInitError(), - "a superseded build's late failure must not clobber the healthy env's initErr") - assert.NotEqual(t, ld.ErrInitializationFailed, env.GetInitError()) - assert.NotNil(t, env.GetClient(), "B's healthy client still serves") - // The superseded build was discarded, not installed for A. - envImpl.mu.RLock() - _, aHasClient := envImpl.clients[envConfig.SDKKey] - envImpl.mu.RUnlock() - assert.False(t, aHasClient, "the superseded build must not be installed for A") -} - // TestReanchor_SupersededLateBuildIsDiscarded: even a *successful* initial build is discarded if a // re-anchor committed a fresh anchor client while it was in flight. Installing it would tear down the // current anchor client (startSDKClient's stale-client guard closes whatever is installed) and swap in diff --git a/internal/relayenv/store_handover_realclient_test.go b/internal/relayenv/store_handover_realclient_test.go index 3e5e3aa4..fda5f10d 100644 --- a/internal/relayenv/store_handover_realclient_test.go +++ b/internal/relayenv/store_handover_realclient_test.go @@ -1,33 +1,23 @@ package relayenv // Verifies the real ld.LDClient's Close() behavior against the SSERelayDataStoreAdapter / -// streamUpdatesStoreWrapper pair, which the fake-client tests could not exercise. This is the single -// remaining piece the fake-client tests could not validate: +// streamUpdatesStoreWrapper pair, which the fake-client tests could not exercise: // // > streamUpdatesStoreWrapper.Close() closes the underlying store. With handover the retiring // > and new clients share one underlying store, so closing the retiring client must NOT close // > it — the adapter (not the client) must own the store's lifecycle. (Not reproducible with // > the fake client; verified here against the real client.) -// -// We answer two questions: -// Q1. Does ld.LDClient.Close() invoke Close() on its data store (the wrapper)? -// Q2. After the wrapper's Close() runs, is the underlying store still usable for reads? -// -// Q1 determines whether store handover is at risk at all. Q2 determines whether the remedy needs to -// gate the wrapper's Close (case A: in-memory Close is destructive) or whether it can stay as-is -// (case B: in-memory Close is a no-op and reads still work). import ( "testing" "time" - "github.com/launchdarkly/ld-relay/v8/internal/store" st "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" + "github.com/launchdarkly/ld-relay/v8/internal/store" ld "github.com/launchdarkly/go-server-sdk/v7" "github.com/launchdarkly/go-server-sdk/v7/ldcomponents" "github.com/launchdarkly/go-server-sdk/v7/subsystems" - "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoreimpl" "github.com/launchdarkly/go-server-sdk/v7/subsystems/ldstoretypes" "github.com/stretchr/testify/assert" @@ -56,20 +46,9 @@ func (c *closeObservingStore) GetAll(k ldstoretypes.DataKind) ([]ldstoretypes.Ke func (c *closeObservingStore) Upsert(k ldstoretypes.DataKind, key string, item ldstoretypes.ItemDescriptor) (bool, error) { return c.inner.Upsert(k, key, item) } -func (c *closeObservingStore) IsInitialized() bool { return c.inner.IsInitialized() } -func (c *closeObservingStore) IsStatusMonitoringEnabled() bool { return c.inner.IsStatusMonitoringEnabled() } - -type closeObservingStoreFactory struct { - observed *closeObservingStore -} - -func (f *closeObservingStoreFactory) Build(ctx subsystems.ClientContext) (subsystems.DataStore, error) { - inner, err := ldcomponents.InMemoryDataStore().Build(ctx) - if err != nil { - return nil, err - } - f.observed = &closeObservingStore{inner: inner} - return f.observed, nil +func (c *closeObservingStore) IsInitialized() bool { return c.inner.IsInitialized() } +func (c *closeObservingStore) IsStatusMonitoringEnabled() bool { + return c.inner.IsStatusMonitoringEnabled() } // realClientUsingAdapter spins up a real ld.LDClient backed by the relay store adapter. Using @@ -88,65 +67,6 @@ func realClientUsingAdapter(t *testing.T, adapter *store.SSERelayDataStoreAdapte return client } -// TestRealClient_CloseInvokesWrapperClose verifies that closing a real ld.LDClient causes the -// wrapped store's Close() to fire. This is the precondition that makes store handover dangerous — -// if Close did not propagate, there would be no lifecycle hazard to design around. -func TestRealClient_CloseInvokesWrapperClose(t *testing.T) { - factory := &closeObservingStoreFactory{} - rec := &recordingStreamUpdates{} - adapter := store.NewSSERelayDataStoreAdapter(factory, rec) - - client := realClientUsingAdapter(t, adapter) - require.NotNil(t, factory.observed, "the adapter must have built the observed store") - require.Equal(t, 0, factory.observed.closeCount, "no Close yet before client.Close") - - wrapper := adapter.GetStore() - require.NoError(t, wrapper.Init(st.AllData)) - require.True(t, wrapper.IsInitialized()) - - require.NoError(t, client.Close()) - - // The headline finding: closing the real client propagates Close() to the underlying store via - // streamUpdatesStoreWrapper.Close(). If this assertion fails, the lifecycle caveat is not a real - // hazard for this combination and the store-handover fix only needs the Build() reuse, not a - // Close() lifecycle change. - assert.Equal(t, 1, factory.observed.closeCount, - "ld.LDClient.Close should propagate to the underlying data store via the wrapper") -} - -// TestRealClient_ReadsAfterCloseAreStillFunctional asks the second question: after Close runs, is -// the underlying in-memory store still usable for Get? The answer tells us whether the store-handover -// fix needs to actually prevent Close (because reads will fail after it) or whether reads coincidentally -// still work (because the in-memory store's Close is effectively a no-op for read behavior). Even -// if reads happen to work, the fix should still gate Close — relying on undocumented "Close is a -// no-op" behavior is brittle and breaks when persistent stores enter the picture. -func TestRealClient_ReadsAfterCloseAreStillFunctional(t *testing.T) { - factory := &closeObservingStoreFactory{} - rec := &recordingStreamUpdates{} - adapter := store.NewSSERelayDataStoreAdapter(factory, rec) - - client := realClientUsingAdapter(t, adapter) - wrapper := adapter.GetStore() - require.NoError(t, wrapper.Init(st.AllData)) - - featureKind := ldstoreimpl.Features() - flagKey := st.Flag1ServerSide.Flag.Key - - got, err := wrapper.Get(featureKind, flagKey) - require.NoError(t, err) - require.NotNil(t, got.Item, "sanity: data is readable before close") - - require.NoError(t, client.Close()) - - // Read after Close. The outcome here is informational, not a pass/fail design gate: - // - If the read succeeds, the in-memory store's Close is effectively a no-op for queries; the fix - // can still safely gate Close to be defensive (persistent stores may differ). - // - If the read fails, the fix MUST gate Close, since the new anchor would observe a broken store. - gotAfter, errAfter := wrapper.Get(featureKind, flagKey) - t.Logf("Get after client.Close: item=%v err=%v initialized=%v", - gotAfter.Item != nil, errAfter, wrapper.IsInitialized()) -} - // TestRealClient_HandoverPreservesUnderlyingStore exercises the production store-handover behavior: // when a second real ld.LDClient is built against the same SSERelayDataStoreAdapter — the re-anchor // case — the adapter hands the existing wrapper (and underlying store) to the new client rather diff --git a/internal/store/store_rebuild_after_close_test.go b/internal/store/store_rebuild_after_close_test.go deleted file mode 100644 index 1512b74b..00000000 --- a/internal/store/store_rebuild_after_close_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package store - -// Regression test for the store-handover refcount contract (re-anchor). -// -// SSERelayDataStoreAdapter.Build reuses whatever wrapper is parked in a.store so a re-anchor can hand -// the populated store to the new client. The hazard: once the wrapper's refCount reaches zero and its -// underlying store is torn down, a later Build must NOT hand that same, now-closed wrapper back to a -// new client (a use-after-close for a persistent store whose Close releases its connection pool). The -// fix marks a fully-closed wrapper and has acquire refuse it, so Build rebuilds a fresh wrapper. -// -// (The re-anchor flow keeps the anchor client permanent, so refCount doesn't reach zero while the env -// is alive today; this guards the wrapper/adapter contract itself against a future caller.) - -import ( - "testing" - - "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" - - "github.com/launchdarkly/go-server-sdk/v7/subsystems" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// freshStoreFactory builds a new underlying store on every Build, mirroring a real DataStore factory -// (mockStoreFactory returns a single fixed instance, which can't model a rebuild). -type freshStoreFactory struct { - built []*mockStore -} - -func (f *freshStoreFactory) Build(_ subsystems.ClientContext) (subsystems.DataStore, error) { - s := &mockStore{realStore: sharedtest.NewInMemoryStore()} - f.built = append(f.built, s) - return s, nil -} - -func TestStoreAdapterRebuildsAfterFullClose(t *testing.T) { - factory := &freshStoreFactory{} - updates := &mockEnvStreamsUpdates{} - adapter := NewSSERelayDataStoreAdapter(factory, updates) - ctx := subsystems.BasicClientContext{} - - // First client builds the wrapper: refCount = 1. - first, err := adapter.Build(ctx) - require.NoError(t, err) - sw1 := first.(*streamUpdatesStoreWrapper) - require.Equal(t, 1, sw1.currentRefCount()) - - // The sole client shuts down: refCount 1 -> 0, wrapper marked closed, underlying store torn down. - require.NoError(t, first.Close()) - require.True(t, factory.built[0].closed, "final Close tears down the underlying store") - - // A subsequent Build (e.g. a later re-anchor) must NOT resurrect the fully-closed wrapper — it - // rebuilds a fresh wrapper backed by a fresh, open store. - second, err := adapter.Build(ctx) - require.NoError(t, err) - sw2 := second.(*streamUpdatesStoreWrapper) - - assert.NotSame(t, sw1, sw2, "adapter must rebuild rather than hand back the torn-down wrapper") - assert.Equal(t, 1, sw2.currentRefCount(), "the fresh wrapper starts at refCount 1") - assert.False(t, sw2.store.(*mockStore).closed, "the fresh wrapper's underlying store is open") - assert.Same(t, sw2, adapter.GetStore(), "the adapter now points at the fresh wrapper") - - // acquire on the fully-closed wrapper refuses, so it can never be resurrected. - assert.False(t, sw1.acquire(), "acquire on a fully-closed wrapper must return false") -} diff --git a/internal/store/store_refcount_test.go b/internal/store/store_refcount_test.go index a6d18cb9..41a1189c 100644 --- a/internal/store/store_refcount_test.go +++ b/internal/store/store_refcount_test.go @@ -85,6 +85,38 @@ func TestWrapperCloseIsIdempotent(t *testing.T) { "Close is idempotent: a second Close must not re-close the underlying store") } +// TestWrapperFullyClosedIsNotResurrected: once the final holder has released the wrapper and torn down +// the underlying store, acquire must refuse it, so a later Build (e.g. a subsequent re-anchor) rebuilds a +// fresh wrapper over a fresh, open store rather than handing back the closed one — a use-after-close for +// a persistent store whose Close releases its connection pool. +// +// (The re-anchor flow keeps the anchor client permanent, so refCount doesn't reach zero while the env is +// alive today; this guards the wrapper/adapter contract itself against a future caller.) +func TestWrapperFullyClosedIsNotResurrected(t *testing.T) { + factory := &countingCloseStoreFactory{} + adapter := NewSSERelayDataStoreAdapter(factory, &mockEnvStreamsUpdates{}) + + first, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + sw1 := first.(*streamUpdatesStoreWrapper) + require.Equal(t, 1, sw1.currentRefCount()) + + // The sole client shuts down: refCount 1 -> 0, wrapper marked closed, underlying store torn down. + require.NoError(t, first.Close()) + require.Equal(t, 1, factory.built[0].closeCount()) + assert.False(t, sw1.acquire(), "acquire on a fully-closed wrapper must return false") + + second, err := adapter.Build(subsystems.BasicClientContext{}) + require.NoError(t, err) + sw2 := second.(*streamUpdatesStoreWrapper) + + assert.NotSame(t, sw1, sw2, "adapter must rebuild rather than hand back the torn-down wrapper") + assert.Equal(t, 1, sw2.currentRefCount(), "the fresh wrapper starts at refCount 1") + assert.Same(t, sw2, adapter.GetStore(), "the adapter now points at the fresh wrapper") + require.Len(t, factory.allBuilt(), 2, "a fresh underlying store was built for the fresh wrapper") + assert.Equal(t, 0, factory.built[1].closeCount(), "the fresh wrapper's underlying store is open") +} + // TestWrapperConcurrentCloseClosesExactlyOnce fires many concurrent Close calls on a single wrapper // (modelling stray/duplicate client Closes arriving at once) and asserts the underlying store is torn // down exactly once. This directly exercises the idempotent early-return branch under -race and is diff --git a/relay/concurrent_keys_auth_test.go b/relay/concurrent_keys_auth_test.go index 717f1e7c..f1d4cfbe 100644 --- a/relay/concurrent_keys_auth_test.go +++ b/relay/concurrent_keys_auth_test.go @@ -164,84 +164,88 @@ func rotatedAnchorRep(newAnchor config.SDKKey, version int) envfactory.Environme } } -// A valid non-anchor key authenticates downstream; one upstream connection serves all accepted keys. +// Accepted-set authentication and per-credential rejection within a multi-key environment: every +// accepted credential authenticates downstream (anchor and non-anchor alike) while the anchor owns the +// one upstream connection, a credential outside the accepted set is rejected, and a key the next payload +// drops from the set stops authenticating. +// +// The RAC case removes its two non-anchor keys by marking them scoped to a view rather than by omitting +// them from the arrays. BuildAcceptedSet filters a view-scoped key, so the desired set it produces — +// and therefore the revocation path — is identical either way, and doing it this way additionally keeps +// the RAC *update* call site's view-scoped WARN under test. Removal by omission on the RAC path is +// covered, with a live stream being torn down, by +// TestConcurrentKeysRAC_ConnectedStreamClosedWhenKeyRevokedByOmission. -func TestConcurrentKeysRAC_NonAnchorKeysAuthenticate(t *testing.T) { +func TestConcurrentKeysRAC_RejectsCredentialsOutsideAcceptedSet(t *testing.T) { + cfg := testAutoConfDefaultConfig + // A short cleanup interval so the expiry ticker runs during the test: a revoked key must be gone + // outright, never left scheduled for a later drop. + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) - autoConfTest(t, testAutoConfDefaultConfig, &putEvent, func(p autoConfTestParams) { + autoConfTest(t, cfg, &putEvent, func(p autoConfTestParams) { // The anchor opens the single upstream client; no second client for the non-anchor key. anchorClient := p.awaitClient() assert.Equal(t, anchorSDKKey, anchorClient.Key) p.shouldNotCreateClient(200 * time.Millisecond) - _ = p.awaitEnvironment(multiKeyEnvID) - - // Every accepted credential authenticates downstream, anchor and non-anchor alike. - p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) - p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") - }) -} - -func TestConcurrentKeysOffline_NonAnchorKeysAuthenticate(t *testing.T) { - offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { - p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) - - anchorClient := p.awaitClient() - assert.Equal(t, anchorSDKKey, anchorClient.Key) - p.shouldNotCreateClient(200 * time.Millisecond) - env := p.awaitEnvironment(multiKeyEnvID) - p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) - p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") - - // Flag data flows through the shared store that the single anchor connection populates. - flags, err := env.GetStore().GetAll(ldstoreimpl.Features()) - require.NoError(t, err) - assert.NotEmpty(t, flags) - }) -} - -// Per-credential rejection within a multi-key environment. - -func TestConcurrentKeysRAC_RejectsCredentialsOutsideAcceptedSet(t *testing.T) { - putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) - autoConfTest(t, testAutoConfDefaultConfig, &putEvent, func(p autoConfTestParams) { - _ = p.awaitClient() - _ = p.awaitEnvironment(multiKeyEnvID) - // (a) Accepted siblings authenticate; a credential outside the accepted set is rejected. p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") p.assertSDKEndpointsAvailability(false, config.SDKKey("sdk-not-accepted"), config.MobileKey("mob-not-accepted"), config.EnvironmentID("env-not-accepted")) - // (b) Remove the extra keys via a patch that carries only the anchor; they must then be - // rejected, while the anchor (still accepted) keeps authenticating. - anchorOnly := multiKeyEnvRep( - []envfactory.ConcurrentKeyRep{{Key: "anchor-sdk", Value: string(anchorSDKKey)}}, - []envfactory.ConcurrentKeyRep{{Key: "anchor-mob", Value: string(anchorMobileKey)}}, + // (b) Drop the extra keys from the accepted set via a patch that marks them scoped to a view; + // they must then be rejected, while the anchor (still accepted) keeps authenticating. + removed := multiKeyEnvRep( + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-sdk", Value: string(anchorSDKKey)}, + {Key: "extra-sdk", Value: string(extraSDKKey), HasViews: true}, + }, + []envfactory.ConcurrentKeyRep{ + {Key: "anchor-mob", Value: string(anchorMobileKey)}, + {Key: "extra-mob", Value: string(extraMobileKey), HasViews: true}, + }, 2, ) - p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(anchorOnly)) + p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(removed)) awaitCredentialRemoved(t, p.relay, extraSDKKey) + awaitCredentialRemoved(t, p.relay, extraMobileKey) p.assertSDKEndpointsAvailability(false, extraSDKKey, extraMobileKey, "") p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) + + // The anchor value never changed, so this was not a re-anchor: it still owns the connection. + assert.Equal(t, anchorSDKKey, env.GetAcceptedKeys().Anchor) + + // The RAC update handler names each rejected identifier so an operator can find it in the UI. + p.mockLog.AssertMessageMatch(t, true, ldlog.Warn, + multiKeyIdentifiers.GetDisplayName()+".*rejecting credentials scoped to a view: extra-sdk, extra-mob") }) } func TestConcurrentKeysOffline_RejectsCredentialsOutsideAcceptedSet(t *testing.T) { offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) - _ = p.awaitClient() - _ = p.awaitEnvironment(multiKeyEnvID) + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) + + env := p.awaitEnvironment(multiKeyEnvID) + + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") p.assertSDKEndpointsAvailability(false, config.SDKKey("sdk-not-accepted"), config.MobileKey("mob-not-accepted"), config.EnvironmentID("env-not-accepted")) + // Flag data flows through the shared store that the single anchor connection populates. + flags, err := env.GetStore().GetAll(ldstoreimpl.Features()) + require.NoError(t, err) + assert.NotEmpty(t, flags) + // Reload with only the anchor accepted; the extra keys are dropped immediately (omitted). p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv( []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}}, @@ -255,63 +259,12 @@ func TestConcurrentKeysOffline_RejectsCredentialsOutsideAcceptedSet(t *testing.T }) } -// A connected SDK is disconnected when its (non-anchor) key expires. +// A key past its expiry stops authenticating. // -// The downstream SDK connects on a non-anchor key while that key is still permanent, so the -// connection establishes independent of expiry timing. We then give the connected key a near-future -// expiry; once the timestamp passes, the cleanup ticker must drop the key AND disconnect the open -// stream. Covered for both an SDK key and a mobile key. (The live open-connection teardown is -// verified on the offline path, which uses a real SDK client that actually serves stream data; the -// RAC equivalent — TestConcurrentKeysRAC_KeyExpiryRemovesCredential — verifies the same expiry->reject -// outcome at the auth layer, since FakeLDClient does not serve stream data.) -func TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyExpires(t *testing.T) { - // The server-side stream (/all) emits "put"; the mobile streams (/meval, /mping) emit "ping". - run := func(t *testing.T, streamPath, firstEvent string, connectKey credential.SDKCredential, expiringSDK bool) { - cfg := config.Config{} - cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) - offlineModeTest(t, cfg, func(p offlineModeTestParams) { - p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) - _ = p.awaitClient() - env := p.awaitEnvironment(multiKeyEnvID) - require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) - - req := sharedtest.BuildRequestWithAuth("GET", streamPath, connectKey, nil) - sharedtest.WithStreamRequest(t, req, p.relay, func(eventCh <-chan eventsource.Event) { - // Confirm the stream is live before we expire the key. - sharedtest.AwaitEventOfType(t, eventCh, firstEvent, 5*time.Second) - - // Give the connected non-anchor key a near-future expiry; keep the anchor permanent. - expiry := time.Now().Add(100 * time.Millisecond) - sdkKeys := []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}, {Value: extraSDKKey}} - mobileKeys := []envfactory.AcceptedMobileKey{{Value: anchorMobileKey}, {Value: extraMobileKey}} - if expiringSDK { - sdkKeys[1].Expiry = expiry - } else { - mobileKeys[1].Expiry = expiry - } - p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv(sdkKeys, mobileKeys)) - - // The cleanup ticker drops the expired key and disconnects this stream. - awaitStreamClosed(t, eventCh, 5*time.Second) - }) - - // After expiry: the dropped key no longer authenticates; the anchor (sibling) still does. - if expiringSDK { - p.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") - } else { - p.assertSDKEndpointsAvailability(false, "", extraMobileKey, "") - } - p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) - }) - } - - t.Run("sdk key", func(t *testing.T) { run(t, "/all", "put", extraSDKKey, true) }) - t.Run("mobile key", func(t *testing.T) { - // base64 of {"key":"userkey","kind":"user"} — a valid context, not the legacy user format. - run(t, "/meval/eyJrZXkiOiJ1c2Vya2V5Iiwia2luZCI6InVzZXIifQ==", "ping", extraMobileKey, false) - }) -} - +// The live open-connection teardown is verified on the offline path, which uses a real SDK client that +// actually serves stream data (TestConcurrentKeysOffline_SiblingStreamSurvivesWhileExpiringKeyDisconnects); +// the RAC equivalent below verifies the same expiry->reject outcome at the auth layer, since FakeLDClient +// does not serve stream data. func TestConcurrentKeysRAC_KeyExpiryRemovesCredential(t *testing.T) { cfg := testAutoConfDefaultConfig cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) @@ -411,35 +364,12 @@ func TestConcurrentKeysRAC_KeyWithFutureExpiryStillAuthenticates(t *testing.T) { // Rotating the anchor in the accepted set. // -// Both tests run on the FakeLDClient harness, so they verify the routing/credential-level behavior -// of an anchor swap. The real-upstream store handover (avoiding an empty-store window) and -// rollback-on-init-failure robustness is covered by the re-anchor tests in internal/relayenv. - -// When a new anchor arrives via RAC (sdkKey.value changes to a brand-new key), the upstream client -// swaps to the new anchor and the old anchor is dropped, while the non-anchor key stays accepted. -func TestConcurrentKeysRAC_RotatingAnchorUpdatesUpstreamClient(t *testing.T) { - putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) - autoConfTest(t, testAutoConfDefaultConfig, &putEvent, func(p autoConfTestParams) { - client1 := p.awaitClient() - assert.Equal(t, anchorSDKKey, client1.Key) - _ = p.awaitEnvironment(multiKeyEnvID) - - // A new anchor arrives; the old anchor is rotated out, the non-anchor extra key is retained. - p.stream.Enqueue(configsource.MakeAutoConfigPatchEvent(rotatedAnchorRep(rotatedAnchorSDKKey, 2))) - - // The new anchor opens the single upstream client; the old anchor's client closes. - client2 := p.awaitClient() - assert.Equal(t, rotatedAnchorSDKKey, client2.Key) - client1.AwaitClose(t, 5*time.Second) - - awaitCredentialRemoved(t, p.relay, anchorSDKKey) - - // The new anchor and the retained non-anchor key authenticate; the old anchor no longer does. - p.assertSDKEndpointsAvailability(true, rotatedAnchorSDKKey, anchorMobileKey, multiKeyEnvID) - p.assertSDKEndpointsAvailability(true, extraSDKKey, "", "") - p.assertSDKEndpointsAvailability(false, anchorSDKKey, "", "") - }) -} +// The credential-level routing outcome of an anchor swap on the FakeLDClient harness — new anchor's +// client opened, old client closed, no client for the non-anchor keys, old anchor dropped — is covered +// by TestConcurrentKeysRAC_MixedUpdateAddsReanchorsAndRemovesInOnePayload, which does the same swap in a +// payload that also adds and removes a key. The real-upstream store handover (avoiding an empty-store +// window) and rollback-on-init-failure robustness are covered by the re-anchor tests in +// internal/relayenv. // A downstream SDK connected on a non-anchor key stays connected when the anchor is rotated out from // under it. This uses a real (dummy) SDK client + RAC mock — rather than the FakeLDClient harness — @@ -494,60 +424,10 @@ func TestConcurrentKeysRAC_NonAnchorConnectionSurvivesAnchorRotation(t *testing. h.assertSDKEndpointsAvailability(true, extraSDKKey, "", "") } -// multiKeyArchiveEnvWithAnchor is multiKeyArchiveEnv with a caller-chosen anchor SDK key, used to -// rotate the anchor across an archive reload. The chosen anchor must also appear in sdkKeys. -func multiKeyArchiveEnvWithAnchor(anchor config.SDKKey, sdkKeys []envfactory.AcceptedSDKKey, mobileKeys []envfactory.AcceptedMobileKey) filedata.ArchiveEnvironment { - env := multiKeyArchiveEnv(sdkKeys, mobileKeys) - env.Params.SDKKey = anchor - return env -} - -// Rotating the anchor via an offline archive reload. -// -// A downstream SDK connected on a non-anchor key keeps its stream when the archive reloads with the -// anchor rotated to a brand-new key: the new anchor authenticates, the old anchor stops, and the -// non-anchor sibling is undisturbed. Offline re-anchoring reuses the environment's single file-data -// client rather than swapping an upstream connection (that swap is the RAC path, exercised by -// TestConcurrentKeysRAC_RotatingAnchorUpdatesUpstreamClient), so no new client is built and the open -// connection survives. The non-anchor expiry-via-reload half of the offline reload-rotation scenario is -// covered by TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyExpires. -func TestConcurrentKeysOffline_AnchorRotationViaArchiveReload(t *testing.T) { - offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { - p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) - anchorClient := p.awaitClient() - assert.Equal(t, anchorSDKKey, anchorClient.Key) - env := p.awaitEnvironment(multiKeyEnvID) - require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) - - // Connect a downstream SDK on the non-anchor key and confirm it is live before rotating. - req := sharedtest.BuildRequestWithAuth("GET", "/all", extraSDKKey, nil) - sharedtest.WithStreamRequest(t, req, p.relay, func(eventCh <-chan eventsource.Event) { - sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) - - // Reload the archive with the anchor rotated to a brand-new key; the non-anchor extra SDK key - // stays accepted and the mobile keys are unchanged. - p.updateHandler.UpdateEnvironment(multiKeyArchiveEnvWithAnchor( - rotatedAnchorSDKKey, - []envfactory.AcceptedSDKKey{{Value: rotatedAnchorSDKKey}, {Value: extraSDKKey}}, - defaultAcceptedMobileKeys(), - )) - - // The offline re-anchor reuses the single file-data client, so the non-anchor stream is not - // torn down by the swap. - assertStreamStaysOpen(t, eventCh, 500*time.Millisecond) - }) - - // Offline re-anchoring commits without building a replacement upstream client. - p.shouldNotCreateClient(200 * time.Millisecond) - - awaitCredentialRemoved(t, p.relay, anchorSDKKey) - - // The rotated anchor and the retained non-anchor key authenticate; the old anchor no longer does. - p.assertSDKEndpointsAvailability(true, rotatedAnchorSDKKey, anchorMobileKey, multiKeyEnvID) - p.assertSDKEndpointsAvailability(true, extraSDKKey, "", "") - p.assertSDKEndpointsAvailability(false, anchorSDKKey, "", "") - }) -} +// Rotating the anchor via an offline archive reload — the new anchor authenticating, the old anchor +// stopping, a retained credential's open stream surviving, and no replacement upstream client being +// built — is covered by TestConcurrentKeysOffline_MixedUpdateAddsReanchorsAndRemovesInOneReload, which +// performs the same reload-with-rotation plus an add and a remove. // awaitStreamClosed reads from a WithStreamRequest event channel until the stream-closed sentinel // (a nil event) arrives, failing if the timeout elapses first. Non-nil events are ignored. @@ -672,67 +552,13 @@ func TestConcurrentKeysOffline_SDKOnlyEnvironmentAuthenticates(t *testing.T) { }) } -// A single patch that adds one non-anchor key and removes another, with the anchor held fixed. -// -// Because the anchor value does not change, there is no re-anchor: the sole upstream client keeps -// serving. The added entry starts routing, the removed entry stops, and a downstream stream that was -// open before the patch is left undisturbed. Uses a real (dummy) client + RAC mock so there is a live -// stream to observe (FakeLDClient never serves a stream body). -func TestConcurrentKeysRAC_ArrayPatchAddsAndRemovesNonAnchorKeys(t *testing.T) { - putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) - racMock := configsource.NewRACMock(t, &putEvent) - - cfg := config.Config{AutoConfig: config.AutoConfigConfig{Key: testAutoConfKey}} - cfg.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(racMock.URL) - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - relay, err := newRelayInternal(cfg, relayInternalOptions{ - loggers: mockLog.Loggers, - clientFactory: testclient.CreateDummyClient, - }) - require.NoError(t, err) - defer relay.Close() - - h := relayTestHelper{t: t, relay: relay} - env := h.awaitEnvironment(multiKeyEnvID) - require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) - - // Hold an open downstream stream on the anchor while the non-anchor entries change around it. - req := sharedtest.BuildRequestWithAuth("GET", "/all", anchorSDKKey, nil) - sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { - sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) - - // One patch that adds a new non-anchor key (addedSDKKey) and removes the existing one - // (extraSDKKey), keeping the anchor and both mobile keys. - patch := multiKeyEnvRep( - []envfactory.ConcurrentKeyRep{ - {Key: "anchor-sdk", Value: string(anchorSDKKey)}, - {Key: "added-sdk", Value: string(addedSDKKey)}, - }, - defaultMobileKeyReps(), - 2, - ) - racMock.Send(configsource.MakeAutoConfigPatchEvent(patch)) - - // The added key routes and the removed key stops routing. - require.Eventually(t, func() bool { - _, errAdded := relay.getEnvironment(sdkauth.New(addedSDKKey)) - _, errRemoved := relay.getEnvironment(sdkauth.New(extraSDKKey)) - return errAdded == nil && errRemoved != nil - }, 5*time.Second, 5*time.Millisecond) - - // The anchor's open stream is undisturbed by the non-anchor add/remove. - assertStreamStaysOpen(t, eventCh, 500*time.Millisecond) - }) - - // The anchor never changed, so no re-anchor happened and the anchor still owns the connection. - assert.Equal(t, anchorSDKKey, env.GetAcceptedKeys().Anchor) - h.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) - h.assertSDKEndpointsAvailability(true, addedSDKKey, "", "") - h.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") -} +// A single patch that adds one non-anchor key and removes another, with the anchor held fixed, is +// covered piecewise: TestConcurrentKeysRAC_MixedUpdateAddsReanchorsAndRemovesInOnePayload owns the +// add/remove routing, and _RejectsCredentialsOutsideAcceptedSet owns "the anchor never changed, so no +// re-anchor happened". That an open stream survives while a *different* credential is removed around it +// is owned by _NonAnchorConnectionSurvivesAnchorRotation: removeCredential cannot tell which credential +// a given open stream belongs to, so "the anchor's stream survives removal of a non-anchor key" and "a +// non-anchor stream survives removal of the old anchor" traverse the same teardown path. // A malformed offline payload preserves the previous credentials and does not reconnect. // diff --git a/relay/concurrent_keys_lifecycle_test.go b/relay/concurrent_keys_lifecycle_test.go index e2783063..9fb0f26f 100644 --- a/relay/concurrent_keys_lifecycle_test.go +++ b/relay/concurrent_keys_lifecycle_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" "github.com/launchdarkly/ld-relay/v8/internal/filedata" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" @@ -138,75 +139,72 @@ func TestConcurrentKeysRAC_ConnectedStreamClosedWhenKeyRevokedByOmission(t *test h.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) } -// The offline-reload twin of the RAC revocation-by-omission case: a key dropped from the reloaded -// archive is revoked immediately and its connected downstream SDK is disconnected, while the retained -// anchor keeps authenticating. -func TestConcurrentKeysOffline_ConnectedStreamClosedWhenKeyRevokedByOmission(t *testing.T) { - offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { - p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) - _ = p.awaitClient() - env := p.awaitEnvironment(multiKeyEnvID) - require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) - - req := sharedtest.BuildRequestWithAuth("GET", "/all", extraSDKKey, nil) - sharedtest.WithStreamRequest(t, req, p.relay, func(eventCh <-chan eventsource.Event) { - sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) - - // Reload with the non-anchor key omitted: it is revoked immediately. - p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv( - []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}}, - defaultAcceptedMobileKeys(), - )) - - awaitStreamClosed(t, eventCh, 5*time.Second) - }) - - awaitCredentialRemoved(t, p.relay, extraSDKKey) - p.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") - p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) - }) -} - -// When one key expires, the disconnect must be targeted: only that key's downstream SDKs drop. A stream -// held on the anchor stays connected throughout the expiry window while a concurrently-open stream on -// the expiring non-anchor key is torn down. Uses the offline harness (real client that serves stream -// data) with two simultaneous downstream connections on the same environment. +// The offline-reload twin of the case above is +// TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyGainsView: "gains a view" and "omitted from +// the array" produce the identical desired-set diff inside BuildAcceptedSet — the key is simply absent — +// so both drive the same immediate-revocation-and-stream-teardown path through the offline update +// handler, and that test covers it for a mobile key as well as an SDK key. + +// When one key expires, the cleanup ticker must drop it and disconnect its downstream SDKs — and only +// its own: a stream held on the anchor stays connected throughout the expiry window while a +// concurrently-open stream on the expiring non-anchor key is torn down. Uses the offline harness (real +// client that serves stream data) with two simultaneous downstream connections on the same environment. +// +// Run for both an SDK key and a mobile key, because the two kinds of downstream stream are torn down by +// different StreamProviders. func TestConcurrentKeysOffline_SiblingStreamSurvivesWhileExpiringKeyDisconnects(t *testing.T) { - cfg := config.Config{} - cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) - offlineModeTest(t, cfg, func(p offlineModeTestParams) { - p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) - _ = p.awaitClient() - env := p.awaitEnvironment(multiKeyEnvID) - require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) - - // Open a stream on the anchor — the sibling that must stay connected. - anchorReq := sharedtest.BuildRequestWithAuth("GET", "/all", anchorSDKKey, nil) - sharedtest.WithStreamRequest(t, anchorReq, p.relay, func(anchorCh <-chan eventsource.Event) { - sharedtest.AwaitEventOfType(t, anchorCh, "put", 5*time.Second) - - // Concurrently open a second stream on the non-anchor key that we will expire. - expiringReq := sharedtest.BuildRequestWithAuth("GET", "/all", extraSDKKey, nil) - sharedtest.WithStreamRequest(t, expiringReq, p.relay, func(expiringCh <-chan eventsource.Event) { - sharedtest.AwaitEventOfType(t, expiringCh, "put", 5*time.Second) - - // Give the non-anchor key a near-future expiry; the anchor stays permanent. - expiry := time.Now().Add(100 * time.Millisecond) - p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv( - []envfactory.AcceptedSDKKey{{Value: anchorSDKKey}, {Value: extraSDKKey, Expiry: expiry}}, - defaultAcceptedMobileKeys(), - )) - - // Across the expiry window the anchor sibling's stream stays open (the expiring key's - // stream is being torn down on its own channel during this same window)... - assertStreamStaysOpen(t, anchorCh, 300*time.Millisecond) - // ...and the expiring key's stream is confirmed disconnected. - awaitStreamClosed(t, expiringCh, 5*time.Second) + // The server-side stream (/all) emits "put"; the mobile streams (/meval, /mping) emit "ping". + run := func(t *testing.T, streamPath, firstEvent string, connectKey credential.SDKCredential, expiringSDK bool) { + cfg := config.Config{} + cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) + offlineModeTest(t, cfg, func(p offlineModeTestParams) { + p.updateHandler.AddEnvironment(multiKeyArchiveEnv(defaultAcceptedSDKKeys(), defaultAcceptedMobileKeys())) + _ = p.awaitClient() + env := p.awaitEnvironment(multiKeyEnvID) + require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + + // Open a stream on the anchor — the sibling that must stay connected. + anchorReq := sharedtest.BuildRequestWithAuth("GET", "/all", anchorSDKKey, nil) + sharedtest.WithStreamRequest(t, anchorReq, p.relay, func(anchorCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, anchorCh, "put", 5*time.Second) + + // Concurrently open a second stream on the non-anchor key that we will expire. The + // connection is established while that key is still permanent, so it does not race the + // expiry timing. + expiringReq := sharedtest.BuildRequestWithAuth("GET", streamPath, connectKey, nil) + sharedtest.WithStreamRequest(t, expiringReq, p.relay, func(expiringCh <-chan eventsource.Event) { + sharedtest.AwaitEventOfType(t, expiringCh, firstEvent, 5*time.Second) + + // Give the connected non-anchor key a near-future expiry; the anchor and primary + // mobile key stay permanent. + expiry := time.Now().Add(100 * time.Millisecond) + sdkKeys := defaultAcceptedSDKKeys() + mobileKeys := defaultAcceptedMobileKeys() + if expiringSDK { + sdkKeys[1].Expiry = expiry + } else { + mobileKeys[1].Expiry = expiry + } + p.updateHandler.UpdateEnvironment(multiKeyArchiveEnv(sdkKeys, mobileKeys)) + + // Across the expiry window the anchor sibling's stream stays open (the expiring key's + // stream is being torn down on its own channel during this same window)... + assertStreamStaysOpen(t, anchorCh, 300*time.Millisecond) + // ...and the expiring key's stream is confirmed disconnected. + awaitStreamClosed(t, expiringCh, 5*time.Second) + }) }) + + // After the expiry: the dropped key no longer authenticates; the anchor sibling still does. + if expiringSDK { + p.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") + } else { + p.assertSDKEndpointsAvailability(false, "", extraMobileKey, "") + } + p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) }) + } - // After the expiry: the dropped key no longer authenticates; the anchor sibling still does. - p.assertSDKEndpointsAvailability(false, extraSDKKey, "", "") - p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) - }) + t.Run("sdk key", func(t *testing.T) { run(t, "/all", "put", extraSDKKey, true) }) + t.Run("mobile key", func(t *testing.T) { run(t, mobileEvalContextPath, "ping", extraMobileKey, false) }) } diff --git a/relay/concurrent_keys_views_test.go b/relay/concurrent_keys_views_test.go index dbf0599a..96ae37fa 100644 --- a/relay/concurrent_keys_views_test.go +++ b/relay/concurrent_keys_views_test.go @@ -24,14 +24,11 @@ import ( "github.com/launchdarkly/ld-relay/v8/internal/api" "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/envfactory" - "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" "github.com/launchdarkly/ld-relay/v8/internal/sdks" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest" "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/configsource" - "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" "github.com/launchdarkly/eventsource" - "github.com/launchdarkly/go-configtypes" "github.com/launchdarkly/go-sdk-common/v3/ldlog" "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" @@ -49,8 +46,8 @@ const ( viewScopedMobID = "view-scoped-mob" ) -// The standard two-entry arrays plus a third entry scoped to a view, in each of the four shapes the -// two harnesses need (wire reps for RAC, accepted-key params for the offline archive). +// The standard two-entry wire-rep arrays plus a third entry scoped to a view, for the RAC harness. The +// offline harness builds its accepted-key params inline, alongside the keys it revokes mid-session. func viewScopedSDKKeyReps() []envfactory.ConcurrentKeyRep { return append(defaultSDKKeyReps(), @@ -62,16 +59,6 @@ func viewScopedMobileKeyReps() []envfactory.ConcurrentKeyRep { envfactory.ConcurrentKeyRep{Key: viewScopedMobID, Value: string(viewScopedMobileKey), HasViews: true}) } -func viewScopedAcceptedSDKKeys() []envfactory.AcceptedSDKKey { - return append(defaultAcceptedSDKKeys(), - envfactory.AcceptedSDKKey{Key: viewScopedSDKID, Value: viewScopedSDKKey, HasViews: true}) -} - -func viewScopedAcceptedMobileKeys() []envfactory.AcceptedMobileKey { - return append(defaultAcceptedMobileKeys(), - envfactory.AcceptedMobileKey{Key: viewScopedMobID, Value: viewScopedMobileKey, HasViews: true}) -} - // assertViewScopedKeysAbsentFromStatus verifies the /status sdkKeys[]/mobileKeys[] arrays do not list // the view-scoped credentials. Those arrays are rendered straight from the accepted set, so this is the // externally-visible proof that a filtered key carries no state anywhere in the environment. @@ -158,30 +145,6 @@ func TestConcurrentKeysRAC_ViewScopedKeysAreRejected(t *testing.T) { }) } -func TestConcurrentKeysOffline_ViewScopedKeysAreRejected(t *testing.T) { - offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { - p.updateHandler.AddEnvironment(multiKeyArchiveEnv( - viewScopedAcceptedSDKKeys(), viewScopedAcceptedMobileKeys())) - - anchorClient := p.awaitClient() - assert.Equal(t, anchorSDKKey, anchorClient.Key) - p.shouldNotCreateClient(200 * time.Millisecond) - - env := p.awaitEnvironment(multiKeyEnvID) - - p.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) - p.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") - p.assertSDKEndpointsAvailability(false, viewScopedSDKKey, viewScopedMobileKey, "") - - accepted := env.GetAcceptedKeys() - assert.NotContains(t, accepted.Server, viewScopedSDKKey) - assert.NotContains(t, accepted.Mobile, viewScopedMobileKey) - - assertViewScopedKeysAbsentFromStatus(t, p.relay) - assertViewScopedKeysRejectedWarning(t, p.mockLog) - }) -} - // A malformed payload that also carries view-scoped keys logs the malformed error and stays silent // about the view-scoped ones. // @@ -215,7 +178,12 @@ func TestConcurrentKeysOffline_MalformedPayloadSuppressesViewScopedWarning(t *te // immediately rather than on an expiry timestamp, and RemoveConnectionMapping unmaps before the streams // are torn down so a reconnect is rejected. These tests assert that behavior rather than build it. The // live teardown is verified on the offline path, which uses a real SDK client that actually serves -// stream data (mirroring TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyExpires). +// stream data (mirroring +// TestConcurrentKeysOffline_SiblingStreamSurvivesWhileExpiringKeyDisconnects). +// +// The initial AddEnvironment payload also carries an already-view-scoped SDK key and mobile key, so this +// covers the offline *add* call site's ingestion filter (never accepted, absent from /status, WARN +// emitted exactly once) alongside the offline *update* call site exercised below. func TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyGainsView(t *testing.T) { // The server-side stream (/all) emits "put"; the mobile streams (/meval, /mping) emit "ping". run := func(t *testing.T, streamPath, firstEvent string, connectKey credential.SDKCredential, viewOnSDK bool) { @@ -243,11 +211,28 @@ func TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyGainsView(t *testi } offlineModeTest(t, config.Config{}, func(p offlineModeTestParams) { - p.updateHandler.AddEnvironment(multiKeyArchiveEnv(named(false))) - _ = p.awaitClient() + // The add payload carries the two named non-anchor keys plus one already-view-scoped SDK key + // and mobile key, so the add handler's ingestion filter is exercised here too. + initialSDK, initialMob := named(false) + p.updateHandler.AddEnvironment(multiKeyArchiveEnv( + append(initialSDK, envfactory.AcceptedSDKKey{Key: viewScopedSDKID, Value: viewScopedSDKKey, HasViews: true}), + append(initialMob, envfactory.AcceptedMobileKey{Key: viewScopedMobID, Value: viewScopedMobileKey, HasViews: true}), + )) + anchorClient := p.awaitClient() + assert.Equal(t, anchorSDKKey, anchorClient.Key) + p.shouldNotCreateClient(200 * time.Millisecond) env := p.awaitEnvironment(multiKeyEnvID) require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) + // The add handler filtered both view-scoped entries: neither is accepted, neither reaches + // /status, and the WARN naming them is emitted exactly once. + p.assertSDKEndpointsAvailability(false, viewScopedSDKKey, viewScopedMobileKey, "") + addAccepted := env.GetAcceptedKeys() + assert.NotContains(t, addAccepted.Server, viewScopedSDKKey) + assert.NotContains(t, addAccepted.Mobile, viewScopedMobileKey) + assertViewScopedKeysAbsentFromStatus(t, p.relay) + assertViewScopedKeysRejectedWarning(t, p.mockLog) + req := sharedtest.BuildRequestWithAuth("GET", streamPath, connectKey, nil) sharedtest.WithStreamRequest(t, req, p.relay, func(eventCh <-chan eventsource.Event) { // Confirm the stream is live before the key gains a view. @@ -294,68 +279,9 @@ func TestConcurrentKeysOffline_ConnectedSDKDisconnectedWhenKeyGainsView(t *testi }) } -// The RAC equivalent, with an open downstream stream held on the anchor while the non-anchor key gains -// a view around it: the revoked key stops routing, a reconnect with it is rejected, and the anchor's -// live connection is untouched. Mirrors TestConcurrentKeysRAC_ArrayPatchAddsAndRemovesNonAnchorKeys. -func TestConcurrentKeysRAC_KeyGainingViewIsRevokedAndOthersUndisturbed(t *testing.T) { - putEvent := configsource.MakeAutoConfigPutEvent(multiKeyEnvRep(defaultSDKKeyReps(), defaultMobileKeyReps(), 1)) - racMock := configsource.NewRACMock(t, &putEvent) - - cfg := config.Config{AutoConfig: config.AutoConfigConfig{Key: testAutoConfKey}} - cfg.Main.StreamURI, _ = configtypes.NewOptURLAbsoluteFromString(racMock.URL) - // A short cleanup interval so the expiry ticker runs during the test: a revoked view-scoped key - // must be gone outright, never left scheduled for a later drop. - cfg.Main.ExpiredCredentialCleanupInterval = configtypes.NewOptDuration(100 * time.Millisecond) - - mockLog := ldlogtest.NewMockLog() - defer mockLog.DumpIfTestFailed(t) - - relay, err := newRelayInternal(cfg, relayInternalOptions{ - loggers: mockLog.Loggers, - clientFactory: testclient.CreateDummyClient, - }) - require.NoError(t, err) - defer relay.Close() - - h := relayTestHelper{t: t, relay: relay} - env := h.awaitEnvironment(multiKeyEnvID) - require.Eventually(t, func() bool { return env.GetClient() != nil }, 5*time.Second, 5*time.Millisecond) - h.assertSDKEndpointsAvailability(true, extraSDKKey, extraMobileKey, "") - - req := sharedtest.BuildRequestWithAuth("GET", "/all", anchorSDKKey, nil) - sharedtest.WithStreamRequest(t, req, relay, func(eventCh <-chan eventsource.Event) { - sharedtest.AwaitEventOfType(t, eventCh, "put", 5*time.Second) - - // One patch marks both non-anchor entries as view-scoped, keeping the anchor and primary. - patch := multiKeyEnvRep( - []envfactory.ConcurrentKeyRep{ - {Key: "anchor-sdk", Value: string(anchorSDKKey)}, - {Key: "extra-sdk", Value: string(extraSDKKey), HasViews: true}, - }, - []envfactory.ConcurrentKeyRep{ - {Key: "anchor-mob", Value: string(anchorMobileKey)}, - {Key: "extra-mob", Value: string(extraMobileKey), HasViews: true}, - }, - 2, - ) - racMock.Send(configsource.MakeAutoConfigPatchEvent(patch)) - - // Both stop routing, so a reconnect presenting either one is rejected. - require.Eventually(t, func() bool { - _, errSDK := relay.getEnvironment(sdkauth.New(extraSDKKey)) - _, errMobile := relay.getEnvironment(sdkauth.New(extraMobileKey)) - return errSDK != nil && errMobile != nil - }, 5*time.Second, 5*time.Millisecond, "keys that gained a view were not revoked") - - // The anchor's open stream is undisturbed by the revocation alongside it. - assertStreamStaysOpen(t, eventCh, 500*time.Millisecond) - }) - - // The anchor never changed, so no re-anchor happened and it still owns the connection. - assert.Equal(t, anchorSDKKey, env.GetAcceptedKeys().Anchor) - h.assertSDKEndpointsAvailability(true, anchorSDKKey, anchorMobileKey, multiKeyEnvID) - h.assertSDKEndpointsAvailability(false, extraSDKKey, extraMobileKey, "") - - mockLog.AssertMessageMatch(t, true, ldlog.Warn, - multiKeyIdentifiers.GetDisplayName()+".*rejecting credentials scoped to a view: extra-sdk, extra-mob") -} +// The RAC update call site's equivalent — a non-anchor key that gains a view stops routing, and the WARN +// names it — is folded into TestConcurrentKeysRAC_RejectsCredentialsOutsideAcceptedSet, whose removal +// patch marks its non-anchor entries view-scoped. That an open stream on a *different* credential +// survives the revocation alongside it is owned by +// TestConcurrentKeysRAC_NonAnchorConnectionSurvivesAnchorRotation: removeCredential cannot tell which +// credential a given open stream belongs to, so both traverse the same teardown path. diff --git a/relay/endpoints_status_test.go b/relay/endpoints_status_test.go index f6371761..d12296fc 100644 --- a/relay/endpoints_status_test.go +++ b/relay/endpoints_status_test.go @@ -339,32 +339,3 @@ func TestEndpointsStatusDuringInFlightRotation(t *testing.T) { require.False(t, findKeyStatusByValue(envStatus2.GetByKey("sdkKeys"), sdks.ObscureKey(string(newAnchor))).IsNull(), "the new anchor must be present in sdkKeys[] after the commit") } - -// TestKeyStatus verifies the helper that converts an accepted key into its status-endpoint JSON form. -func TestKeyStatus(t *testing.T) { - strptr := func(s string) *string { return &s } - - t.Run("permanent key with identifier", func(t *testing.T) { - ks := keyStatus("sdk-abc123", credential.AcceptedKey{Key: strptr("default")}) - assert.Equal(t, "default", ks.Key) - assert.Equal(t, sdks.ObscureKey("sdk-abc123"), ks.Value) - assert.Nil(t, ks.Expiry) - }) - - t.Run("nil identifier yields empty Key (omitted in JSON)", func(t *testing.T) { - ks := keyStatus("sdk-legacy", credential.AcceptedKey{Key: nil}) - assert.Equal(t, "", ks.Key) - }) - - t.Run("expiring key has expiry in Unix milliseconds", func(t *testing.T) { - expiry := time.Date(2099, 6, 1, 12, 0, 0, 0, time.UTC) - ks := keyStatus("sdk-old", credential.AcceptedKey{Key: strptr("old-key"), Expiry: &expiry}) - require.NotNil(t, ks.Expiry) - assert.Equal(t, expiry.UnixMilli(), *ks.Expiry) - }) - - t.Run("mobile key value is obscured", func(t *testing.T) { - ks := keyStatus("mob-secret", credential.AcceptedKey{Key: strptr("mob-1")}) - assert.Equal(t, sdks.ObscureKey("mob-secret"), ks.Value) - }) -} diff --git a/relay/relay_endpoints_test.go b/relay/relay_endpoints_test.go index 222ed631..92ef17ac 100644 --- a/relay/relay_endpoints_test.go +++ b/relay/relay_endpoints_test.go @@ -62,18 +62,8 @@ func TestReportFlagEvalRejectsOversizedBodyWhenLimitConfigured(t *testing.T) { assert.Equal(t, http.StatusRequestEntityTooLarge, resp.Code) } -func TestReportFlagEvalAllowsLargeBodyWhenNoLimitConfigured(t *testing.T) { - headers := make(http.Header) - headers.Set("Content-Type", "application/json") - ctx := testenv.NewTestEnvContext("", false, st.MakeStoreWithData(true)) - - req := buildPreRoutedRequest("REPORT", jsonhelpers.ToJSON(st.BasicUserForTestFlags), headers, nil, ctx) - resp := httptest.NewRecorder() - evaluateAllFeatureFlags(basictypes.JSClientSDK, ct.OptBase2Bytes{})(resp, req) - - assert.Equal(t, http.StatusOK, resp.Code) -} - +// An unset body limit (ct.OptBase2Bytes{}) is not enforced: the test below issues the same REPORT with +// no limit configured and gets a 200, so it covers that branch as well as the body it asserts on. func TestReportFlagEvalWorksWithUninitializedClientButInitializedStore(t *testing.T) { headers := make(http.Header) headers.Set("Content-Type", "application/json") From 53ad7d9cf43142fef4b90faf4fcbd12d293024f8 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Mon, 17 Aug 2026 12:17:20 -0700 Subject: [PATCH 62/66] fix(autoconfigcache): bump the cached model version to 2 (#822) Bumps the auto-config cache model version from 1 to 2, so a relay built from this branch rejects cache entries written by a v8 relay instead of accepting them as its own. --- internal/autoconfigcache/model.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/autoconfigcache/model.go b/internal/autoconfigcache/model.go index 56a60bbd..ba29b9c1 100644 --- a/internal/autoconfigcache/model.go +++ b/internal/autoconfigcache/model.go @@ -17,7 +17,7 @@ const ( // CurrentModelVersion is the version of the serialization format. // Increment this when the shape of EnvironmentRep or FilterRep changes. -const CurrentModelVersion = 1 +const CurrentModelVersion = 2 // CachedItem is the versioned envelope stored in the cache. It wraps the actual data // with kind and version metadata so we can detect and handle format changes on read. From c3cd0a91ea62c698cffd45013581e1be1faaf92b Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 18 Aug 2026 07:32:48 -0700 Subject: [PATCH 63/66] fix(relayenv): reject revoked credentials in GetStreamHandler (#828) --- internal/credential/rotator.go | 25 ++++ internal/credential/rotator_test.go | 106 +++++++++++++++ .../env_context_handler_fanout_test.go | 27 +++- internal/relayenv/env_context_impl.go | 15 +- ..._context_stream_handler_revocation_test.go | 128 ++++++++++++++++++ 5 files changed, 296 insertions(+), 5 deletions(-) create mode 100644 internal/relayenv/env_context_stream_handler_revocation_test.go diff --git a/internal/credential/rotator.go b/internal/credential/rotator.go index 544a0e36..72862184 100644 --- a/internal/credential/rotator.go +++ b/internal/credential/rotator.go @@ -149,6 +149,31 @@ func (r *Rotator) AllCredentials() []SDKCredential { return r.allCredentials() } +// IsAccepted reports whether cred is currently one of the environment's accepted credentials: an +// accepted SDK key, an accepted mobile key (including one carrying a future expiry, which still +// authenticates until the cleanup ticker drops it), or the environment ID. Any other credential type is +// never accepted. +// +// This answers the same question as membership in AllCredentials, by direct map lookup, so callers on +// request paths do not allocate a credential slice per call. +func (r *Rotator) IsAccepted(cred SDKCredential) bool { + r.mu.RLock() + defer r.mu.RUnlock() + + switch cred := cred.(type) { + case config.SDKKey: + _, ok := r.acceptedSDKKeys[cred] + return ok + case config.MobileKey: + _, ok := r.acceptedMobileKeys[cred] + return ok + case config.EnvironmentID: + return r.primaryEnvironmentID.Defined() && cred == r.primaryEnvironmentID + default: + return false + } +} + func (r *Rotator) expireSDKKey(sdkKey config.SDKKey) { r.loggers.Infof("Deprecated SDK key %s has expired and is no longer valid for authentication", sdkKey.Masked()) delete(r.acceptedSDKKeys, sdkKey) diff --git a/internal/credential/rotator_test.go b/internal/credential/rotator_test.go index 5103a73c..a56f4be1 100644 --- a/internal/credential/rotator_test.go +++ b/internal/credential/rotator_test.go @@ -571,3 +571,109 @@ func TestRevertAnchorChangeDoesNotAdmitUndefinedPreviousAnchor(t *testing.T) { } assert.NotContains(t, r.AllCredentials(), SDKCredential(keyB), "failed new anchor dropped") } + +func TestIsAcceptedMatchesAllCredentials(t *testing.T) { + // IsAccepted is the membership form of AllCredentials, so the two must never disagree -- including on + // keys carrying a future expiry, which still authenticate until the cleanup ticker drops them. + r := newTestRotator() + anchor := config.SDKKey("anchor") + expiringSDK := config.SDKKey("expiring-sdk") + mob := config.MobileKey("mob") + expiringMobile := config.MobileKey("expiring-mob") + envID := config.EnvironmentID("env-id") + now := time.Unix(1000, 0) + + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: expiringSDK, Expiry: util.PtrOrNil(now.Add(time.Hour))}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob}). + WithMobileKey(MobileKeyParams{Value: expiringMobile, Expiry: util.PtrOrNil(now.Add(time.Hour))}). + WithEnvironmentID(envID)), + now) + r.StepTime(now) + + all := r.AllCredentials() + require.ElementsMatch(t, []SDKCredential{anchor, expiringSDK, mob, expiringMobile, envID}, all) + for _, cred := range all { + assert.True(t, r.IsAccepted(cred), "AllCredentials reported %s, so IsAccepted must agree", cred.Masked()) + } + + // Values of each tracked kind that were never accepted. + assert.False(t, r.IsAccepted(config.SDKKey("unknown-sdk"))) + assert.False(t, r.IsAccepted(config.MobileKey("unknown-mob"))) + assert.False(t, r.IsAccepted(config.EnvironmentID("unknown-env"))) + + // Undefined values are never accepted -- the rotator only ever holds defined credentials. + assert.False(t, r.IsAccepted(config.SDKKey(""))) + assert.False(t, r.IsAccepted(config.MobileKey(""))) + assert.False(t, r.IsAccepted(config.EnvironmentID(""))) + + // A credential kind the rotator does not track is never accepted, even when defined. + assert.False(t, r.IsAccepted(config.AutoConfigKey("auto-config-key"))) +} + +func TestIsAcceptedFollowsRevocation(t *testing.T) { + // Reconciling to a set that omits a key revokes it, and IsAccepted must report it as such while + // leaving the retained credentials accepted. This is the predicate the stream handler relies on to + // reject a credential revoked after the request authenticated. + r := newTestRotator() + anchor := config.SDKKey("anchor") + other := config.SDKKey("other") + mob := config.MobileKey("mob") + otherMob := config.MobileKey("other-mob") + now := time.Unix(1000, 0) + + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: other}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob}). + WithMobileKey(MobileKeyParams{Value: otherMob})), + now) + r.StepTime(now) + require.True(t, r.IsAccepted(other)) + require.True(t, r.IsAccepted(otherMob)) + + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob})), + now) + r.StepTime(now) + + assert.False(t, r.IsAccepted(other), "revoked SDK key") + assert.False(t, r.IsAccepted(otherMob), "revoked mobile key") + assert.True(t, r.IsAccepted(anchor), "anchor is retained") + assert.True(t, r.IsAccepted(mob), "primary mobile key is retained") +} + +func TestIsAcceptedFalseOnceExpiryElapses(t *testing.T) { + // A key carrying a future expiry stays accepted until the cleanup ticker drops it, then stops being + // accepted -- without any further reconcile. + r := newTestRotator() + anchor := config.SDKKey("anchor") + expiringSDK := config.SDKKey("expiring-sdk") + mob := config.MobileKey("mob") + expiringMobile := config.MobileKey("expiring-mob") + now := time.Unix(1000, 0) + expiry := now.Add(time.Hour) + + r.Reconcile( + mustBuild(t, NewAcceptedSetBuilder(). + WithAnchor(SDKKeyParams{Value: anchor}). + WithSDKKey(SDKKeyParams{Value: expiringSDK, Expiry: util.PtrOrNil(expiry)}). + WithPrimaryMobileKey(MobileKeyParams{Value: mob}). + WithMobileKey(MobileKeyParams{Value: expiringMobile, Expiry: util.PtrOrNil(expiry)})), + now) + r.StepTime(now) + require.True(t, r.IsAccepted(expiringSDK), "accepted before its expiry elapses") + require.True(t, r.IsAccepted(expiringMobile), "accepted before its expiry elapses") + + r.StepTime(expiry.Add(time.Second)) + + assert.False(t, r.IsAccepted(expiringSDK), "dropped by the cleanup ticker") + assert.False(t, r.IsAccepted(expiringMobile), "dropped by the cleanup ticker") + assert.True(t, r.IsAccepted(anchor), "the anchor never expires") + assert.True(t, r.IsAccepted(mob), "the primary mobile key is permanent") +} diff --git a/internal/relayenv/env_context_handler_fanout_test.go b/internal/relayenv/env_context_handler_fanout_test.go index 66758bd0..8a35b89a 100644 --- a/internal/relayenv/env_context_handler_fanout_test.go +++ b/internal/relayenv/env_context_handler_fanout_test.go @@ -5,8 +5,9 @@ package relayenv // build the handler on demand, scoping it with the env's (immutable) filter key. These tests exercise // that on-demand path directly: that the provider is asked for the right scoped credential, that a valid // credential yields the provider's handler, and that a credential the provider rejects (wrong kind) falls -// back to the 404 handler. End-to-end multi-key streaming through the full HTTP stack is covered by the -// relay-package concurrent-keys auth suite. +// back to the 404 handler. Rejection of a credential that is no longer accepted is covered in +// env_context_stream_handler_revocation_test.go. End-to-end multi-key streaming through the full HTTP +// stack is covered by the relay-package concurrent-keys auth suite. import ( "net/http" @@ -14,6 +15,7 @@ import ( "testing" "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/credential" "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" "github.com/launchdarkly/ld-relay/v8/internal/streams" @@ -23,6 +25,15 @@ import ( "github.com/stretchr/testify/require" ) +// rotatorAccepting returns a rotator whose accepted set is exactly creds. GetStreamHandler consults the +// accepted set before asking the provider for a handler, so a test envContextImpl needs a real rotator +// rather than the zero value. +func rotatorAccepting(creds ...credential.SDKCredential) *credential.Rotator { + r := credential.NewRotator(ldlog.NewDisabledLoggers()) + r.Initialize(creds) + return r +} + // fakeStreamProvider records the scoped credential passed to Handler and returns a caller-supplied // handler for credentials it accepts (nil otherwise, mimicking a real provider rejecting the wrong // credential kind). Register/Close are unused by GetStreamHandler. @@ -56,7 +67,10 @@ func TestGetStreamHandler_BuildsOnDemandScopedWithEnvFilterKey(t *testing.T) { }, } - c := &envContextImpl{filterKey: filter} + c := &envContextImpl{ + filterKey: filter, + keyRotator: rotatorAccepting(config.SDKKey("sdk-A"), config.SDKKey("sdk-B")), + } // A valid (right-kind) credential: the provider is asked for that credential scoped with the env's // filter key, and its handler is returned as-is (no per-credential storage, built on the spot). @@ -86,7 +100,12 @@ func TestGetStreamHandler_WrongKindCredentialServes404(t *testing.T) { }, } - c := &envContextImpl{filterKey: config.DefaultFilter} + // The mobile key is deliberately in the accepted set: this test is about the provider rejecting the + // wrong credential kind, so the accepted-set check must pass in order for the request to reach it. + c := &envContextImpl{ + filterKey: config.DefaultFilter, + keyRotator: rotatorAccepting(config.MobileKey("mob-key")), + } // A credential the provider rejects (returns nil for) must fall back to the invalid-stream 404 // handler, exactly as the old per-credential map miss did. diff --git a/internal/relayenv/env_context_impl.go b/internal/relayenv/env_context_impl.go index 60146096..fff5b9a5 100644 --- a/internal/relayenv/env_context_impl.go +++ b/internal/relayenv/env_context_impl.go @@ -507,7 +507,7 @@ func (c *envContextImpl) startSDKClient(sdkKey config.SDKKey, readyCh chan<- Env // sdkKeyIsActive reports whether the rotator still accepts sdkKey. startSDKClient uses this to avoid // installing a client for a key revoked while the client was building. func (c *envContextImpl) sdkKeyIsActive(sdkKey config.SDKKey) bool { - return slices.Contains(c.keyRotator.AllCredentials(), credential.SDKCredential(sdkKey)) + return c.keyRotator.IsAccepted(sdkKey) } func (c *envContextImpl) GetPayloadFilter() config.FilterKey { @@ -825,6 +825,19 @@ func (c *envContextImpl) GetLoggers() ldlog.Loggers { } func (c *envContextImpl) GetStreamHandler(streamProvider streams.StreamProvider, cred credential.SDKCredential) http.Handler { + // Re-check the accepted set. The middleware authenticates the credential once, at the start of the + // request, and a credential can be revoked while that request is still in flight: on the REPORT + // stream endpoints the client paces the body read that precedes this call. The providers only + // type-check the credential, so a revoked one would otherwise get a working handler, not a 404. + // + // A revocation can still land between this check and the subscription registering. The eventsource + // handler writes the status line first, so that case still returns 200. + // + // The rotator guards its own accepted set. Do not take c.mu here: reanchor holds it for writing + // across its whole commit sequence, and stream connects must not queue behind it. + if !c.keyRotator.IsAccepted(cred) { + return http.HandlerFunc(invalidStreamHandler) + } // Build the handler on demand: every handler in a (filter, provider) slot differs only by the // credential-derived channel id. c.filterKey is immutable after construction, so this needs no lock. if h := streamProvider.Handler(sdkauth.NewScoped(c.filterKey, cred)); h != nil { diff --git a/internal/relayenv/env_context_stream_handler_revocation_test.go b/internal/relayenv/env_context_stream_handler_revocation_test.go new file mode 100644 index 00000000..09ea1b46 --- /dev/null +++ b/internal/relayenv/env_context_stream_handler_revocation_test.go @@ -0,0 +1,128 @@ +package relayenv + +// GetStreamHandler consults the rotator's accepted set before asking the StreamProvider for a handler. +// The middleware authenticates the request's credential once, up front, so a credential revoked while the +// request is still in flight would otherwise be handed a working stream. These tests drive revocation +// through the real Rotator.Reconcile path, and pair every expected 404 with a still-accepted control +// credential -- without the control, a handler that was simply broken for everything would pass. + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/credential" + "github.com/launchdarkly/ld-relay/v8/internal/sdkauth" + + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// streamServedCode is the status the test provider's handler writes. It is deliberately not 200, so a +// served stream can never be confused with a status written by anything else in the chain. +const streamServedCode = 299 + +// alwaysServingProvider returns a handler for every credential kind, so any 404 in these tests can only +// have come from the accepted-set check -- never from a provider declining the credential's kind. +func alwaysServingProvider() *fakeStreamProvider { + return &fakeStreamProvider{ + handlerFn: func(sdkauth.ScopedCredential) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(streamServedCode) } + }, + } +} + +// serveCode resolves cred through GetStreamHandler and reports the status its handler writes. +func serveCode(t *testing.T, c *envContextImpl, sp *fakeStreamProvider, cred credential.SDKCredential) int { + t.Helper() + rr := httptest.NewRecorder() + c.GetStreamHandler(sp, cred).ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/", nil)) + return rr.Code +} + +func TestGetStreamHandlerRejectsRevokedCredentials(t *testing.T) { + anchor := config.SDKKey("anchor-sdk-key") + revokedSDK := config.SDKKey("revoked-sdk-key") + primaryMobile := config.MobileKey("primary-mob-key") + revokedMobile := config.MobileKey("revoked-mob-key") + envID := config.EnvironmentID("env-id") + now := time.Unix(1000, 0) + + r := credential.NewRotator(ldlog.NewDisabledLoggers()) + full, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: anchor}). + WithSDKKey(credential.SDKKeyParams{Value: revokedSDK}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: primaryMobile}). + WithMobileKey(credential.MobileKeyParams{Value: revokedMobile}). + WithEnvironmentID(envID). + Build() + require.NoError(t, err) + + result := r.Reconcile(full, now) + require.NotNil(t, result.AnchorChange, "the first anchor is signaled as a change") + r.CommitAnchor(result.AnchorChange.NewAnchor) + r.StepTime(now) + + c := &envContextImpl{filterKey: config.DefaultFilter, keyRotator: r} + sp := alwaysServingProvider() + + // Baseline: every accepted credential kind reaches the provider's handler. + assert.Equal(t, streamServedCode, serveCode(t, c, sp, anchor), "anchor") + assert.Equal(t, streamServedCode, serveCode(t, c, sp, revokedSDK), "SDK key, before revocation") + assert.Equal(t, streamServedCode, serveCode(t, c, sp, primaryMobile), "primary mobile key") + assert.Equal(t, streamServedCode, serveCode(t, c, sp, revokedMobile), "mobile key, before revocation") + assert.Equal(t, streamServedCode, serveCode(t, c, sp, envID), "environment ID") + + // Revoke the two non-primary keys outright by reconciling to a set that omits them. + reduced, err := credential.NewAcceptedSetBuilder(). + WithAnchor(credential.SDKKeyParams{Value: anchor}). + WithPrimaryMobileKey(credential.MobileKeyParams{Value: primaryMobile}). + WithEnvironmentID(envID). + Build() + require.NoError(t, err) + r.Reconcile(reduced, now) + r.StepTime(now) + + assert.Equal(t, http.StatusNotFound, serveCode(t, c, sp, revokedSDK), "revoked SDK key") + assert.Equal(t, http.StatusNotFound, serveCode(t, c, sp, revokedMobile), "revoked mobile key") + + // The retained credentials are untouched, so the 404s above are attributable to the revocation. + assert.Equal(t, streamServedCode, serveCode(t, c, sp, anchor), "anchor is still accepted") + assert.Equal(t, streamServedCode, serveCode(t, c, sp, primaryMobile), "primary mobile key is still accepted") + assert.Equal(t, streamServedCode, serveCode(t, c, sp, envID), "environment ID is still accepted") +} + +func TestGetStreamHandlerDoesNotConsultProviderForRevokedCredential(t *testing.T) { + // The check short-circuits before the provider is asked for a handler, so a revoked credential can + // never reach the point of creating a channel subscription. + c := &envContextImpl{ + filterKey: config.DefaultFilter, + keyRotator: rotatorAccepting(config.SDKKey("accepted-sdk-key")), + } + sp := alwaysServingProvider() + + assert.Equal(t, http.StatusNotFound, serveCode(t, c, sp, config.SDKKey("revoked-sdk-key"))) + assert.Empty(t, sp.scopes, "the provider must not be asked to build a handler for a revoked credential") + + // The accepted credential still goes through, and only then is the provider consulted. + assert.Equal(t, streamServedCode, serveCode(t, c, sp, config.SDKKey("accepted-sdk-key"))) + require.Len(t, sp.scopes, 1) + assert.Equal(t, config.SDKKey("accepted-sdk-key"), sp.scopes[0].SDKCredential) +} + +func TestGetStreamHandlerRejectsForeignEnvironmentID(t *testing.T) { + // JS client streams authenticate with the environment ID, so it goes through the same check: only this + // environment's own ID is accepted. + c := &envContextImpl{ + filterKey: config.DefaultFilter, + keyRotator: rotatorAccepting(config.EnvironmentID("this-env")), + } + sp := alwaysServingProvider() + + assert.Equal(t, streamServedCode, serveCode(t, c, sp, config.EnvironmentID("this-env"))) + assert.Equal(t, http.StatusNotFound, serveCode(t, c, sp, config.EnvironmentID("other-env"))) +} From b59a1f81bfb1aad333e2c586b4352392807dcde2 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 18 Aug 2026 09:35:42 -0700 Subject: [PATCH 64/66] feat(projmanager): add a queue for auto-config actions per environment AutoConfigActionQueue wraps AutoConfigActions so that each environment's actions run on their own goroutine, in submission order, independently of every other environment's. Nothing uses it yet; the next commit wires it into Relay. A worker exists only while its queue holds work. nextAction decides between taking the next action and retiring the queue, and it makes that decision under the same lock enqueue uses for lookup -- so an action can never land on a retired queue, and two workers can never run for one environment. The queue key includes the filter key, because a filtered environment is a separate context with its own SDK client and its own anchor. ReceivedAllEnvironments is a drain barrier: it forwards only once every environment with outstanding work has drained, so a readiness signal cannot fire while an environment is still being applied. It derives that set from the live queues, since a queue is removed only after its worker has run everything in it. It waits on its own goroutine, so the caller is never blocked. A semaphore bounds concurrent actions, and execute recovers from a panic -- an escaped panic would kill the worker and leave its queue in the map with nothing draining it, silently freezing that environment's configuration. The wrapped handler is named next, following the usual convention for a wrapper in a chain: q.next.AddEnvironment reads as delegation rather than recursion. Also widen the package doc, which described only routing by project key and did not mention the interfaces the package owns. --- .../projmanager/autoconfig_action_queue.go | 274 +++++++++ .../autoconfig_action_queue_test.go | 563 ++++++++++++++++++ internal/projmanager/package_info.go | 7 +- 3 files changed, 842 insertions(+), 2 deletions(-) create mode 100644 internal/projmanager/autoconfig_action_queue.go create mode 100644 internal/projmanager/autoconfig_action_queue_test.go diff --git a/internal/projmanager/autoconfig_action_queue.go b/internal/projmanager/autoconfig_action_queue.go new file mode 100644 index 00000000..9188631f --- /dev/null +++ b/internal/projmanager/autoconfig_action_queue.go @@ -0,0 +1,274 @@ +package projmanager + +import ( + "sync" + "time" + + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" +) + +var _ AutoConfigActions = &AutoConfigActionQueue{} + +const ( + // defaultMaxConcurrentActions bounds how many environment actions execute at once. See + // AutoConfigActionQueue.execute for why the bound exists; it is set well above any realistic + // environment count for a single account so that ordinary rotations still overlap freely. + defaultMaxConcurrentActions = 32 + + // defaultCloseGracePeriod is how long Close waits for queued actions to finish. Waiting + // indefinitely would hand back the shutdown delay this type exists to remove: an environment + // whose anchor build is stalled holds its action for up to Main.InitTimeout (10s by default), + // and the process should not sit on that. Abandoning in-flight actions is safe -- see Close. + defaultCloseGracePeriod = 2 * time.Second +) + +// envKey identifies one relay environment for queueing purposes. The filter key is part of +// the identity, not decoration: EnvironmentManager turns one upstream environment into one default +// plus one per configured filter (see EnvironmentParams.WithFilter), and each of those is a separate +// EnvContext with its own SDK client and its own anchor. They must not wait behind each other. +type envKey struct { + envID config.EnvironmentID + filter config.FilterKey +} + +func keyForParams(params envfactory.EnvironmentParams) envKey { + return envKey{envID: params.EnvID, filter: params.Identifiers.FilterKey} +} + +// envQueue holds the actions still to run for one environment, oldest first. It carries no lock of +// its own: AutoConfigActionQueue.mu guards every access, which is what makes the retire-vs-enqueue +// handoff in drain race-free. +type envQueue struct { + pending []func() +} + +// AutoConfigActionQueue wraps an AutoConfigActions so that each environment's actions run on their +// own goroutine, in submission order, independently of every other environment's. +// +// Why this exists: StreamManager consumes the auto-configuration SSE stream on a single goroutine +// and ran each environment's action to completion inline. An environment whose SDK anchor moves +// rebuilds its upstream client synchronously (relayenv's reanchor -> buildNewAnchorClient), blocking +// for up to Main.InitTimeout. Every other environment's adds, updates, and credential revocations +// queued behind it, so a bulk rotation across N environments cost N * InitTimeout in the worst case +// and a revocation could sit unapplied for minutes. One queue per environment makes that cost the +// slowest single build rather than the sum of all of them. +// +// What is deliberately preserved: +// +// - Per-environment ordering. Actions for one environment share a queue, so an add followed by a +// delete for that environment can never reorder. Only distinct environments overlap. +// - StreamManager's single-message-at-a-time invariant. Only the handler call is deferred; event +// parsing, version bookkeeping (envReceiver.Upsert) and cache writes still happen on the stream +// goroutine before anything is queued here. +// - The re-anchor sequence itself. Nothing in relayenv changes: within an environment the +// reconcile is still fully synchronous, so its commit/rollback atomicity holds as before. +// +// Running distinct environments concurrently is safe because the relay-wide state they touch is +// already synchronized -- Relay.addEnvironment holds Relay.lock, and EnvironmentLookup has its own +// mutex -- and because each environment already runs a credential-expiry ticker goroutine that +// mutates that same state concurrently with the stream goroutine. This extends a concurrency class +// the design already accommodates rather than introducing a new one. +type AutoConfigActionQueue struct { + next AutoConfigActions + loggers ldlog.Loggers + sem chan struct{} + closeGrace time.Duration + mu sync.Mutex + queues map[envKey]*envQueue + closed bool + outstanding sync.WaitGroup +} + +// NewAutoConfigActionQueue wraps next so that environments are processed independently of one +// another. The caller must Close it once no further actions can arrive. +func NewAutoConfigActionQueue(next AutoConfigActions, loggers ldlog.Loggers) *AutoConfigActionQueue { + return newAutoConfigActionQueue(next, loggers, defaultMaxConcurrentActions, defaultCloseGracePeriod) +} + +func newAutoConfigActionQueue( + next AutoConfigActions, + loggers ldlog.Loggers, + maxConcurrent int, + closeGrace time.Duration, +) *AutoConfigActionQueue { + loggers.SetPrefix("[AutoConfigActionQueue]") + return &AutoConfigActionQueue{ + next: next, + loggers: loggers, + sem: make(chan struct{}, maxConcurrent), + closeGrace: closeGrace, + queues: make(map[envKey]*envQueue), + } +} + +// The three environment methods share one shape: work out which environment the call concerns, then +// queue the same call against the next handler. Only the queue key differs. +func (q *AutoConfigActionQueue) AddEnvironment(params envfactory.EnvironmentParams) { + q.enqueue(keyForParams(params), func() { q.next.AddEnvironment(params) }) +} + +func (q *AutoConfigActionQueue) UpdateEnvironment(params envfactory.EnvironmentParams) { + q.enqueue(keyForParams(params), func() { q.next.UpdateEnvironment(params) }) +} + +func (q *AutoConfigActionQueue) DeleteEnvironment(id config.EnvironmentID, filter config.FilterKey) { + q.enqueue(envKey{envID: id, filter: filter}, func() { q.next.DeleteEnvironment(id, filter) }) +} + +// ReceivedAllEnvironments forwards to the wrapped handler only once every action queued since the +// previous call has run. That keeps the downstream meaning of the signal intact: it sets Relay's +// fullyConfigured flag, which gates both request serving (getEnvironment returns errRelayNotReady +// until it is set) and the reported status health, so it must not fire while environments from the +// payload are still being applied. +// +// It returns immediately and waits on its own goroutine. Blocking the caller would hand head-of-line +// blocking back for "put" payloads specifically, which is the case this type is most needed for. +// +// Note this makes readiness slightly stricter than it was: previously the flag was set as soon as +// the payload had been walked, which on a first "put" is before any environment's client has been +// built. It now additionally waits for each environment's action to have run. +func (q *AutoConfigActionQueue) ReceivedAllEnvironments() { + q.mu.Lock() + if q.closed { + q.mu.Unlock() + return + } + // Wait on every environment that still has work outstanding. A queue leaves q.queues only after + // its worker has run everything in it -- drain pops an action, runs it, and only deletes the queue + // on a later pass that finds it empty -- so an environment absent here is already fully applied + // and needs no barrier. + pending := make([]envKey, 0, len(q.queues)) + for key := range q.queues { + pending = append(pending, key) + } + q.mu.Unlock() + + var drained sync.WaitGroup + drained.Add(len(pending)) + for _, key := range pending { + if !q.enqueue(key, drained.Done) { + // Closed underneath us; nothing will run this barrier, so release it here rather than + // leaving the goroutine below parked forever. + drained.Done() + } + } + + q.outstanding.Go(func() { + drained.Wait() + q.mu.Lock() + closed := q.closed + q.mu.Unlock() + if !closed { + q.next.ReceivedAllEnvironments() + } + }) +} + +// Close stops accepting new actions and waits up to the grace period for queued ones to finish. +// +// Actions still running when the grace period expires are abandoned rather than waited on, which is +// safe because every one of them is already guarded against a closing Relay: addEnvironment refuses +// once Relay is closed, EnvContext.Close is idempotent, and the re-anchor sequence re-checks the +// environment's closed flag after its build and declines to commit into a closed environment. +func (q *AutoConfigActionQueue) Close() { + q.mu.Lock() + if q.closed { + q.mu.Unlock() + return + } + q.closed = true + q.mu.Unlock() + + done := make(chan struct{}) + go func() { + q.outstanding.Wait() + close(done) + }() + + select { + case <-done: + case <-time.After(q.closeGrace): + q.loggers.Warnf("Timed out after %v waiting for in-flight environment actions to finish; abandoning them", q.closeGrace) + } +} + +// enqueue appends fn to key's queue, starting a worker for that environment if one is not already +// running. It reports whether the action was accepted; it is refused only after Close. It never +// blocks on the action itself, so the caller (the stream goroutine) is never delayed by one +// environment's work. +func (q *AutoConfigActionQueue) enqueue(key envKey, fn func()) bool { + q.mu.Lock() + defer q.mu.Unlock() + + if q.closed { + return false + } + envq := q.queues[key] + if envq == nil { + envq = &envQueue{} + q.queues[key] = envq + q.outstanding.Go(func() { q.drain(key, envq) }) + } + envq.pending = append(envq.pending, fn) + return true +} + +// drain runs key's actions one at a time until the queue empties, then retires it. A worker per +// non-empty queue (rather than one per environment for the environment's whole life) means a +// deleted or idle environment leaves nothing behind to leak. +func (q *AutoConfigActionQueue) drain(key envKey, envq *envQueue) { + for { + fn, ok := q.nextAction(key, envq) + if !ok { + return + } + q.execute(fn) + } +} + +// nextAction takes the oldest action off key's queue. When the queue is empty it retires the queue +// instead and reports false, which ends the worker. +// +// Deciding between those two outcomes under the same lock that enqueue uses to look queues up is what +// makes the handoff safe. A concurrent enqueue either finds this queue and appends -- so the emptiness +// check below sees the new action -- or it creates a fresh queue after the delete. An action can never +// land on a retired queue, and two workers can never run for one environment. +func (q *AutoConfigActionQueue) nextAction(key envKey, envq *envQueue) (func(), bool) { + q.mu.Lock() + defer q.mu.Unlock() + + if len(envq.pending) == 0 { + if q.queues[key] == envq { + delete(q.queues, key) + } + return nil, false + } + fn := envq.pending[0] + envq.pending[0] = nil // release the closure; the slice header still points past it + envq.pending = envq.pending[1:] + return fn, true +} + +// execute runs a single action under the concurrency bound. +// +// The bound exists because a "put" that adds every environment at once would otherwise construct +// them all in parallel -- each one configuring a data store and registering metrics -- where +// previously they were built one at a time on the stream goroutine. It also caps the goroutines an +// account with very many environments can put in flight at once. +// +// A panicking action is contained rather than allowed to kill the worker: an escaped panic would +// leave the queue in the map with nothing draining it, silently freezing that environment's +// configuration for the life of the process. +func (q *AutoConfigActionQueue) execute(fn func()) { + q.sem <- struct{}{} + defer func() { + <-q.sem + if p := recover(); p != nil { + q.loggers.Errorf("Panic while applying an auto-configuration action: %+v", p) + } + }() + fn() +} diff --git a/internal/projmanager/autoconfig_action_queue_test.go b/internal/projmanager/autoconfig_action_queue_test.go new file mode 100644 index 00000000..cd2c22a5 --- /dev/null +++ b/internal/projmanager/autoconfig_action_queue_test.go @@ -0,0 +1,563 @@ +package projmanager + +import ( + "fmt" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/envfactory" + "github.com/launchdarkly/ld-relay/v8/internal/relayenv" +) + +// recordingActions is an AutoConfigActions that records the order in which it was called and can be +// made to block inside a specific environment's action. +type recordingActions struct { + mu sync.Mutex + calls []string + allCount int + + // gate, if non-nil for an environment ID, is waited on inside that environment's action. + gates map[config.EnvironmentID]chan struct{} + // entered is closed-per-env when that environment's action begins. + entered map[config.EnvironmentID]chan struct{} + panicOn config.EnvironmentID +} + +func newRecordingActions() *recordingActions { + return &recordingActions{ + gates: make(map[config.EnvironmentID]chan struct{}), + entered: make(map[config.EnvironmentID]chan struct{}), + } +} + +// gateEnv makes the given environment's actions block until the returned release func is called. +// The second return value is closed once the environment's action has actually started. +func (r *recordingActions) gateEnv(id config.EnvironmentID) (release func(), entered <-chan struct{}) { + gate := make(chan struct{}) + started := make(chan struct{}) + r.mu.Lock() + r.gates[id] = gate + r.entered[id] = started + r.mu.Unlock() + return func() { close(gate) }, started +} + +func (r *recordingActions) enter(id config.EnvironmentID, label string) { + r.mu.Lock() + r.calls = append(r.calls, label) + gate := r.gates[id] + started := r.entered[id] + shouldPanic := r.panicOn == id + r.mu.Unlock() + + if started != nil { + select { + case <-started: + default: + close(started) + } + } + if shouldPanic { + panic(fmt.Sprintf("boom in %s", id)) + } + if gate != nil { + <-gate + } +} + +func (r *recordingActions) AddEnvironment(params envfactory.EnvironmentParams) { + r.enter(params.EnvID, "add:"+string(params.EnvID)+":"+string(params.Identifiers.FilterKey)) +} + +func (r *recordingActions) UpdateEnvironment(params envfactory.EnvironmentParams) { + r.enter(params.EnvID, "update:"+string(params.EnvID)+":"+string(params.Identifiers.FilterKey)) +} + +func (r *recordingActions) DeleteEnvironment(id config.EnvironmentID, filter config.FilterKey) { + r.enter(id, "delete:"+string(id)+":"+string(filter)) +} + +func (r *recordingActions) ReceivedAllEnvironments() { + r.mu.Lock() + r.allCount++ + r.calls = append(r.calls, "all") + r.mu.Unlock() +} + +func (r *recordingActions) recorded() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.calls...) +} + +func (r *recordingActions) receivedAllCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.allCount +} + +func envParams(id config.EnvironmentID, filter config.FilterKey) envfactory.EnvironmentParams { + return envfactory.EnvironmentParams{ + EnvID: id, + Identifiers: relayenv.EnvIdentifiers{ + EnvKey: string(id), + ProjKey: "proj", + FilterKey: filter, + }, + } +} + +func newTestQueue(t *testing.T, maxConcurrent int) (*AutoConfigActionQueue, *recordingActions, *ldlogtest.MockLog) { + t.Helper() + mockLog := ldlogtest.NewMockLog() + t.Cleanup(func() { mockLog.DumpIfTestFailed(t) }) + inner := newRecordingActions() + q := newAutoConfigActionQueue(inner, mockLog.Loggers, maxConcurrent, time.Second) + t.Cleanup(q.Close) + return q, inner, mockLog +} + +// TestAutoConfigActionQueue_SlowEnvironmentDoesNotBlockOthers is the regression test for the defect this +// type exists to fix: an environment stuck rebuilding its SDK client must not delay an unrelated +// environment's update. Before this queue existed, env B's update ran on the same +// goroutine as env A's action and could not start until A returned. +func TestAutoConfigActionQueue_SlowEnvironmentDoesNotBlockOthers(t *testing.T) { + q, inner, _ := newTestQueue(t, defaultMaxConcurrentActions) + + envA := config.EnvironmentID("env-a") + envB := config.EnvironmentID("env-b") + + releaseA, aStarted := inner.gateEnv(envA) + _, bStarted := inner.gateEnv(envB) + + // Env A's action blocks, standing in for a stalled anchor build. + q.UpdateEnvironment(envParams(envA, config.DefaultFilter)) + select { + case <-aStarted: + case <-time.After(time.Second): + require.FailNow(t, "env A's action never started") + } + + // Env B's unrelated update must be processed while A is still stuck. + q.UpdateEnvironment(envParams(envB, config.DefaultFilter)) + select { + case <-bStarted: + case <-time.After(time.Second): + require.FailNow(t, "env B was blocked behind env A's stalled action") + } + + releaseA() +} + +// TestAutoConfigActionQueue_SameEnvironmentStaysOrdered is the other half of the contract: cross- +// environment independence must not cost intra-environment ordering, because RAC actions for one +// environment are order-dependent (an add followed by a delete must not reorder). +func TestAutoConfigActionQueue_SameEnvironmentStaysOrdered(t *testing.T) { + q, inner, _ := newTestQueue(t, defaultMaxConcurrentActions) + + envA := config.EnvironmentID("env-a") + releaseA, aStarted := inner.gateEnv(envA) + + q.AddEnvironment(envParams(envA, config.DefaultFilter)) + select { + case <-aStarted: + case <-time.After(time.Second): + require.FailNow(t, "env A's first action never started") + } + + // Queue two more actions for the same environment while the first is still running. + q.UpdateEnvironment(envParams(envA, config.DefaultFilter)) + q.DeleteEnvironment(envA, config.DefaultFilter) + + // Nothing else for env A may have run yet. + assert.Equal(t, []string{"add:env-a:"}, inner.recorded()) + + releaseA() + + require.Eventually(t, func() bool { + return len(inner.recorded()) == 3 + }, time.Second, 5*time.Millisecond) + assert.Equal(t, []string{"add:env-a:", "update:env-a:", "delete:env-a:"}, inner.recorded()) +} + +// TestAutoConfigActionQueue_FilteredEnvironmentsAreIndependent covers the queue-key choice: a filtered +// environment is a separate EnvContext with its own SDK client and its own anchor, so it must not +// wait behind the default environment that shares its environment ID. +func TestAutoConfigActionQueue_FilteredEnvironmentsAreIndependent(t *testing.T) { + q, inner, _ := newTestQueue(t, defaultMaxConcurrentActions) + + envA := config.EnvironmentID("env-a") + releaseA, aStarted := inner.gateEnv(envA) + + // The default environment blocks. + q.UpdateEnvironment(envParams(envA, config.DefaultFilter)) + select { + case <-aStarted: + case <-time.After(time.Second): + require.FailNow(t, "the default environment's action never started") + } + + // The same environment ID under a filter key is a different environment, so it must proceed. The + // gate is keyed by environment ID, so releasing it also releases this one; assert on the record + // rather than on a second gate. + q.UpdateEnvironment(envParams(envA, config.FilterKey("filter-1"))) + require.Eventually(t, func() bool { + return slices.Contains(inner.recorded(), "update:env-a:filter-1") + }, time.Second, 5*time.Millisecond, "the filtered environment was blocked behind the default one") + + releaseA() +} + +// TestAutoConfigActionQueue_ReceivedAllEnvironmentsWaitsForQueuedWork protects the readiness contract: +// the signal sets Relay's fullyConfigured flag, which gates request serving, so it must not fire +// while environments from the same payload are still being applied. +func TestAutoConfigActionQueue_ReceivedAllEnvironmentsWaitsForQueuedWork(t *testing.T) { + q, inner, _ := newTestQueue(t, defaultMaxConcurrentActions) + + envA := config.EnvironmentID("env-a") + releaseA, aStarted := inner.gateEnv(envA) + + q.AddEnvironment(envParams(envA, config.DefaultFilter)) + q.AddEnvironment(envParams(config.EnvironmentID("env-b"), config.DefaultFilter)) + select { + case <-aStarted: + case <-time.After(time.Second): + require.FailNow(t, "env A's action never started") + } + + q.ReceivedAllEnvironments() + + // It must not have fired: env A is still in flight. + time.Sleep(50 * time.Millisecond) + assert.Zero(t, inner.receivedAllCount(), "readiness fired while an environment was still being applied") + + releaseA() + + require.Eventually(t, func() bool { + return inner.receivedAllCount() == 1 + }, time.Second, 5*time.Millisecond, "readiness never fired after the queued work drained") +} + +// TestAutoConfigActionQueue_ReceivedAllEnvironmentsDoesNotBlockCaller confirms the barrier is asynchronous. +// Blocking the caller would hand head-of-line blocking back for "put" payloads, which is the case +// this type is most needed for. +func TestAutoConfigActionQueue_ReceivedAllEnvironmentsDoesNotBlockCaller(t *testing.T) { + q, inner, _ := newTestQueue(t, defaultMaxConcurrentActions) + + envA := config.EnvironmentID("env-a") + releaseA, aStarted := inner.gateEnv(envA) + defer releaseA() + + q.AddEnvironment(envParams(envA, config.DefaultFilter)) + select { + case <-aStarted: + case <-time.After(time.Second): + require.FailNow(t, "env A's action never started") + } + + returned := make(chan struct{}) + go func() { + q.ReceivedAllEnvironments() + close(returned) + }() + select { + case <-returned: + case <-time.After(time.Second): + require.FailNow(t, "ReceivedAllEnvironments blocked on a stalled environment") + } +} + +// TestAutoConfigActionQueue_ReceivedAllEnvironmentsWithNoQueuedWork covers the common steady-state case: +// a "put" whose environments were all applied already must still forward the signal. +func TestAutoConfigActionQueue_ReceivedAllEnvironmentsWithNoQueuedWork(t *testing.T) { + q, inner, _ := newTestQueue(t, defaultMaxConcurrentActions) + + q.AddEnvironment(envParams(config.EnvironmentID("env-a"), config.DefaultFilter)) + require.Eventually(t, func() bool { + return len(inner.recorded()) == 1 + }, time.Second, 5*time.Millisecond) + + q.ReceivedAllEnvironments() + require.Eventually(t, func() bool { + return inner.receivedAllCount() == 1 + }, time.Second, 5*time.Millisecond) + + // A second signal with nothing touched in between must also forward. + q.ReceivedAllEnvironments() + require.Eventually(t, func() bool { + return inner.receivedAllCount() == 2 + }, time.Second, 5*time.Millisecond) +} + +// TestAutoConfigActionQueue_QueuesAreRetired guards against a goroutine leak per environment. Queues are +// created on demand and must be reaped once drained, or a long-lived Relay with churning +// environments accumulates a worker for every environment it has ever seen. +func TestAutoConfigActionQueue_QueuesAreRetired(t *testing.T) { + q, inner, _ := newTestQueue(t, defaultMaxConcurrentActions) + + for i := range 20 { + id := config.EnvironmentID(fmt.Sprintf("env-%d", i)) + q.AddEnvironment(envParams(id, config.DefaultFilter)) + q.DeleteEnvironment(id, config.DefaultFilter) + } + + require.Eventually(t, func() bool { + return len(inner.recorded()) == 40 + }, 2*time.Second, 5*time.Millisecond) + + require.Eventually(t, func() bool { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.queues) == 0 + }, time.Second, 5*time.Millisecond, "drained queues were not retired") +} + +// TestAutoConfigActionQueue_QueueRecreatedAfterRetirement exercises the retire/submit handoff: an action +// submitted after a queue has been reaped must start a fresh worker rather than be appended to an +// abandoned queue and silently lost. +func TestAutoConfigActionQueue_QueueRecreatedAfterRetirement(t *testing.T) { + q, inner, _ := newTestQueue(t, defaultMaxConcurrentActions) + + envA := config.EnvironmentID("env-a") + + q.AddEnvironment(envParams(envA, config.DefaultFilter)) + require.Eventually(t, func() bool { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.queues) == 0 + }, time.Second, 5*time.Millisecond) + + q.UpdateEnvironment(envParams(envA, config.DefaultFilter)) + require.Eventually(t, func() bool { + return len(inner.recorded()) == 2 + }, time.Second, 5*time.Millisecond, "an action submitted after the queue retired was lost") + assert.Equal(t, []string{"add:env-a:", "update:env-a:"}, inner.recorded()) +} + +// TestAutoConfigActionQueue_ConcurrencyIsBounded confirms the fan-out cap. Without it, a "put" that adds +// every environment at once would construct them all in parallel, where previously they were built +// one at a time on the stream goroutine. +func TestAutoConfigActionQueue_ConcurrencyIsBounded(t *testing.T) { + const bound = 3 + const envCount = 12 + + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + + var inFlight, maxInFlight atomic.Int64 + release := make(chan struct{}) + blocking := &funcActions{ + add: func(envfactory.EnvironmentParams) { + cur := inFlight.Add(1) + for { + old := maxInFlight.Load() + if cur <= old || maxInFlight.CompareAndSwap(old, cur) { + break + } + } + <-release + inFlight.Add(-1) + }, + } + + q := newAutoConfigActionQueue(blocking, mockLog.Loggers, bound, time.Second) + defer q.Close() + + for i := range envCount { + q.AddEnvironment(envParams(config.EnvironmentID(fmt.Sprintf("env-%d", i)), config.DefaultFilter)) + } + + require.Eventually(t, func() bool { + return inFlight.Load() == bound + }, time.Second, 5*time.Millisecond, "expected the bound to be saturated") + + // Give any unbounded work a chance to appear before asserting the ceiling held. + time.Sleep(50 * time.Millisecond) + assert.LessOrEqual(t, maxInFlight.Load(), int64(bound), "more actions ran at once than the bound allows") + + close(release) +} + +// TestAutoConfigActionQueue_PanicDoesNotFreezeEnvironment covers the containment in execute: an escaped +// panic would kill the worker while leaving its queue in the map, silently freezing that +// environment's configuration for the life of the process. +func TestAutoConfigActionQueue_PanicDoesNotFreezeEnvironment(t *testing.T) { + q, inner, mockLog := newTestQueue(t, defaultMaxConcurrentActions) + + envA := config.EnvironmentID("env-a") + inner.mu.Lock() + inner.panicOn = envA + inner.mu.Unlock() + + q.AddEnvironment(envParams(envA, config.DefaultFilter)) + + // Wait for the recovery log rather than for the recorded call. The recording handler appends the + // call before it panics, so the call is visible before execute's deferred recover has run -- and + // asserting on the log at that point races the recover. + require.Eventually(t, func() bool { + for _, line := range mockLog.GetOutput(ldlog.Error) { + if strings.Contains(line, "Panic while applying an auto-configuration action") { + return true + } + } + return false + }, time.Second, 5*time.Millisecond, "the panic was not caught and logged") + + // The environment must still accept work: the queue was not left orphaned. + inner.mu.Lock() + inner.panicOn = "" + inner.mu.Unlock() + + q.UpdateEnvironment(envParams(envA, config.DefaultFilter)) + require.Eventually(t, func() bool { + return len(inner.recorded()) == 2 + }, time.Second, 5*time.Millisecond, "the environment stopped processing actions after a panic") +} + +// TestAutoConfigActionQueue_CloseRefusesNewWork ensures actions arriving after Close are dropped rather +// than applied against a Relay that is tearing down. +func TestAutoConfigActionQueue_CloseRefusesNewWork(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + inner := newRecordingActions() + q := newAutoConfigActionQueue(inner, mockLog.Loggers, defaultMaxConcurrentActions, time.Second) + + q.Close() + q.AddEnvironment(envParams(config.EnvironmentID("env-a"), config.DefaultFilter)) + q.ReceivedAllEnvironments() + + time.Sleep(50 * time.Millisecond) + assert.Empty(t, inner.recorded(), "actions were applied after Close") + assert.Zero(t, inner.receivedAllCount()) + + // Close is idempotent. + q.Close() +} + +// TestAutoConfigActionQueue_CloseWaitsForQueuedWork confirms the ordinary shutdown path drains rather +// than abandons, so a normal Relay shutdown still finishes applying what it accepted. +func TestAutoConfigActionQueue_CloseWaitsForQueuedWork(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + inner := newRecordingActions() + q := newAutoConfigActionQueue(inner, mockLog.Loggers, defaultMaxConcurrentActions, time.Second) + + for i := range 10 { + q.AddEnvironment(envParams(config.EnvironmentID(fmt.Sprintf("env-%d", i)), config.DefaultFilter)) + } + q.Close() + + assert.Len(t, inner.recorded(), 10, "Close returned before queued actions finished") +} + +// TestAutoConfigActionQueue_CloseDoesNotWaitForeverOnStalledWork is the shutdown half of the fix. An +// environment stuck in its anchor build must not hold the process open: Close abandons it once the +// grace period expires, which is safe because the actions re-check Relay's and the environment's +// closed state. +func TestAutoConfigActionQueue_CloseDoesNotWaitForeverOnStalledWork(t *testing.T) { + mockLog := ldlogtest.NewMockLog() + defer mockLog.DumpIfTestFailed(t) + inner := newRecordingActions() + q := newAutoConfigActionQueue(inner, mockLog.Loggers, defaultMaxConcurrentActions, 50*time.Millisecond) + + envA := config.EnvironmentID("env-a") + releaseA, aStarted := inner.gateEnv(envA) + defer releaseA() + + q.AddEnvironment(envParams(envA, config.DefaultFilter)) + select { + case <-aStarted: + case <-time.After(time.Second): + require.FailNow(t, "env A's action never started") + } + + closed := make(chan struct{}) + go func() { + q.Close() + close(closed) + }() + select { + case <-closed: + case <-time.After(2 * time.Second): + require.FailNow(t, "Close blocked indefinitely on a stalled action") + } + mockLog.AssertMessageMatch(t, true, ldlog.Warn, "Timed out.*waiting for in-flight environment actions") +} + +// TestAutoConfigActionQueue_ReceivedAllEnvironmentsWaitsForEarlierWork pins the barrier's scope. The +// barrier waits on every environment that still has work outstanding, not only on the environments +// seen since the previous signal. That is deliberate: the signal drives Relay's readiness flag, and an +// environment that is still being applied is a reason to wait rather than to report ready. +func TestAutoConfigActionQueue_ReceivedAllEnvironmentsWaitsForEarlierWork(t *testing.T) { + q, inner, _ := newTestQueue(t, defaultMaxConcurrentActions) + + envA := config.EnvironmentID("env-a") + releaseA, aStarted := inner.gateEnv(envA) + + // Env A's action starts and stays in flight for the rest of the test. + q.AddEnvironment(envParams(envA, config.DefaultFilter)) + select { + case <-aStarted: + case <-time.After(time.Second): + require.FailNow(t, "env A's action never started") + } + + // A first signal must not fire: env A is outstanding. + q.ReceivedAllEnvironments() + time.Sleep(50 * time.Millisecond) + require.Zero(t, inner.receivedAllCount(), "the first signal fired while env A was in flight") + + // A second signal, concerning a different environment, must still wait for env A. + q.AddEnvironment(envParams(config.EnvironmentID("env-b"), config.DefaultFilter)) + q.ReceivedAllEnvironments() + time.Sleep(50 * time.Millisecond) + require.Zero(t, inner.receivedAllCount(), "a later signal must still wait for env A's outstanding work") + + // Once env A drains, both signals forward. + releaseA() + require.Eventually(t, func() bool { + return inner.receivedAllCount() == 2 + }, time.Second, 5*time.Millisecond, "both signals should forward once all work drains") +} + +// funcActions is an AutoConfigActions whose behavior is supplied per method; unset methods no-op. +type funcActions struct { + add func(envfactory.EnvironmentParams) + update func(envfactory.EnvironmentParams) + del func(config.EnvironmentID, config.FilterKey) + receivedAll func() +} + +func (f *funcActions) AddEnvironment(params envfactory.EnvironmentParams) { + if f.add != nil { + f.add(params) + } +} + +func (f *funcActions) UpdateEnvironment(params envfactory.EnvironmentParams) { + if f.update != nil { + f.update(params) + } +} + +func (f *funcActions) DeleteEnvironment(id config.EnvironmentID, filter config.FilterKey) { + if f.del != nil { + f.del(id, filter) + } +} + +func (f *funcActions) ReceivedAllEnvironments() { + if f.receivedAll != nil { + f.receivedAll() + } +} diff --git a/internal/projmanager/package_info.go b/internal/projmanager/package_info.go index a8c56fcd..c6047eb2 100644 --- a/internal/projmanager/package_info.go +++ b/internal/projmanager/package_info.go @@ -1,3 +1,6 @@ -// Package projmanager contains functionality to execute commands (such as creating, updating, or deleting) -// entities (such as environments and filters) based on inspection of the entity's project key. +// Package projmanager executes the commands that the auto-configuration stream delivers: it creates, +// updates, and deletes entities (environments and filters), routing each one by its project key. It +// also owns the interfaces that describe those commands (AutoConfigActions, EnvironmentActions) and +// the queue that keeps one environment's slow command from delaying another's +// (AutoConfigActionQueue). package projmanager From 0d84ef5543d466e12ad2814fa63e36c2a1f3de18 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 18 Aug 2026 09:35:42 -0700 Subject: [PATCH 65/66] fix(autoconfig): process each environment on its own queue One goroutine reads the auto-configuration stream and, before this change, completed every environment's action inline. An environment whose SDK anchor moves rebuilds its upstream client synchronously, blocking for up to Main.InitTimeout, so one environment's rebuild delayed every other environment's updates and credential revocations. A bulk rotation across N environments cost N * InitTimeout in the worst case, and a revocation issued during one could sit unapplied for minutes. Wrap the action handler in AutoConfigActionQueue. Within an environment the reconcile is still fully synchronous, so the re-anchor's commit/rollback atomicity is unchanged; only distinct environments now overlap. StreamManager still processes one stream message at a time, since only the handler call is deferred. All 20 TestReanchor tests pass without modification. Old-format payloads synthesize the accepted-key model from the singular sdkKey field, so this affected ordinary single-key rotation too, not only environments using concurrent keys. Also rewrite the comment in AddEnvironment that justified its check-then-act by asserting a single calling goroutine. That is now false for distinct environments. The conclusion still holds, for a different reason: the check is keyed per environment, and addEnvironment holds the lock to insert. --- relay/autoconfig_actions.go | 5 +++-- relay/relay.go | 13 ++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/relay/autoconfig_actions.go b/relay/autoconfig_actions.go index 54bffecf..11fb8403 100644 --- a/relay/autoconfig_actions.go +++ b/relay/autoconfig_actions.go @@ -39,8 +39,9 @@ type relayAutoConfigActions struct { func (a *relayAutoConfigActions) AddEnvironment(params envfactory.EnvironmentParams) { // Since we're not holding the lock on the RelayCore, there is theoretically a race condition here // where an environment could be added from elsewhere after we checked in AddOrUpdateEnvironment. - // But in reality, this method is only going to be called from a single goroutine in the auto-config - // stream handler. + // In practice it cannot happen. Calls for one environment share a queue (see + // projmanager.AutoConfigActionQueue), so they never overlap. Calls for different environments do + // run together, but each one checks a different key, and addEnvironment holds the lock to insert. envConfig := envfactory.NewEnvConfigFactoryForAutoConfig(a.r.config.AutoConfig).MakeEnvironmentConfig(params) env, _, err := a.r.addEnvironment(params.Identifiers, envConfig, nil) if err != nil { diff --git a/relay/relay.go b/relay/relay.go index 7b25c617..eff75032 100644 --- a/relay/relay.go +++ b/relay/relay.go @@ -67,6 +67,7 @@ type Relay struct { closed bool lock sync.RWMutex autoConfigStream *autoconfig.StreamManager + autoConfigActions *projmanager.AutoConfigActionQueue archiveManager filedata.ArchiveManagerInterface config config.Config loggers ldlog.Loggers @@ -204,7 +205,12 @@ func newRelayInternal(c config.Config, options relayInternalOptions) (*Relay, er return nil, err } - projectRouter := projmanager.NewProjectRouter(&relayAutoConfigActions{r: r}, loggers) + // Give each environment its own queue rather than sharing one: the actions below can block for up to + // Main.InitTimeout when an environment's SDK anchor moves, and running them inline on the + // stream-consuming goroutine made one environment's rebuild delay every other environment's + // updates and credential revocations. + r.autoConfigActions = projmanager.NewAutoConfigActionQueue(&relayAutoConfigActions{r: r}, loggers) + projectRouter := projmanager.NewProjectRouter(r.autoConfigActions, loggers) r.autoConfigStream = autoconfig.NewStreamManager( c.AutoConfig.Key, @@ -330,6 +336,11 @@ func (r *Relay) Close() error { if r.autoConfigStream != nil { r.autoConfigStream.Close() } + // Closed after the stream, so no further actions can be queued, and before the environments are + // closed below, so queued actions get a chance to finish against a live environment map. + if r.autoConfigActions != nil { + r.autoConfigActions.Close() + } if r.archiveManager != nil { _ = r.archiveManager.Close() } From 90f9026dd8ad20a13ff60c5b38dece27f0d528b2 Mon Sep 17 00:00:00 2001 From: Aaron Zeisler Date: Tue, 18 Aug 2026 09:35:42 -0700 Subject: [PATCH 66/66] test(autoconfig): cover head-of-line blocking between environments Drives two environments through the real auto-configuration handler: environment 1's re-anchor build parks in the client factory, and environment 2's unrelated rotation must still complete while it is parked. Fails in about a second with a clear assertion when the queue is bypassed. Both environments rotate via the singular sdkKey field with no expiring slot, deliberately: that is the shape an ordinary auto-configured account sends, and it still moves the anchor. --- relay/autoconfig_head_of_line_test.go | 102 ++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 relay/autoconfig_head_of_line_test.go diff --git a/relay/autoconfig_head_of_line_test.go b/relay/autoconfig_head_of_line_test.go new file mode 100644 index 00000000..243fa843 --- /dev/null +++ b/relay/autoconfig_head_of_line_test.go @@ -0,0 +1,102 @@ +package relay + +// End-to-end regression coverage for head-of-line blocking in auto-configuration processing, driven +// through the real RAC handler. Intra-environment ordering — the guarantee that must survive making +// environments independent — is covered at the unit level by +// TestSerializedActions_SameEnvironmentStaysOrdered. + +import ( + "sync" + "testing" + "time" + + "github.com/launchdarkly/ld-relay/v8/config" + "github.com/launchdarkly/ld-relay/v8/internal/sdks" + "github.com/launchdarkly/ld-relay/v8/internal/sharedtest/testclient" + + ld "github.com/launchdarkly/go-server-sdk/v7" + helpers "github.com/launchdarkly/go-test-helpers/v3" + + "github.com/stretchr/testify/require" +) + +// TestAutoConfigSlowReanchorDoesNotDelayOtherEnvironments is the regression test for the +// head-of-line blocking defect: an environment whose anchor rotation is stuck building its upstream +// SDK client must not delay an unrelated environment's rotation. +// +// The mechanism being guarded: StreamManager consumes the auto-configuration stream on one +// goroutine, and the re-anchor triggered by a rotation builds its client synchronously, blocking for +// up to Main.InitTimeout. Running those actions inline on the stream goroutine meant every other +// environment's updates — including credential revocations — waited behind it, so a bulk rotation +// across N environments cost N * InitTimeout in the worst case. +// +// Note both environments here rotate via the singular sdkKey field with no expiring slot, i.e. an +// ordinary single-key rotation. That shape is synthesized into the accepted-key model and so still +// moves the anchor, which is why this blocking affects any auto-configured environment and not only +// ones using concurrent keys. +func TestAutoConfigSlowReanchorDoesNotDelayOtherEnvironments(t *testing.T) { + env1Rotated := makeEnvWithModifiedSDKKey(testAutoConfEnv1) + env2Rotated := makeEnvWithModifiedSDKKey(testAutoConfEnv2) + + // Env 1's re-anchor build parks in the factory until released, standing in for a slow or + // unreachable upstream. buildStarted reports that it has actually begun blocking, so the test + // never races ahead of the stall it depends on. + // + // The release is deferred inside the test body below as well as called explicitly, and must run + // before the harness closes the relay: a stalled build holds the environment's reconcile lock, + // which the credential-cleanup ticker also takes, and EnvContext.Close waits for that ticker + // goroutine to exit. Leaving the gate shut on an assertion failure would hang the shutdown instead + // of reporting the failure. + gate := make(chan struct{}) + releaseGate := sync.OnceFunc(func() { close(gate) }) + buildStarted := make(chan struct{}, 1) + + makeFactory := func(createdCh chan<- *testclient.FakeLDClient) sdks.ClientFactoryFunc { + healthy := testclient.FakeLDClientFactoryWithChannel(true, createdCh) + return func(key config.SDKKey, cfg ld.Config, timeout time.Duration) (sdks.LDClientContext, error) { + if key == env1Rotated.SDKKey() { + select { + case buildStarted <- struct{}{}: + default: + } + <-gate + } + return healthy(key, cfg, timeout) + } + } + + initialEvent := makeAutoConfPutEvent(testAutoConfEnv1, testAutoConfEnv2) + autoConfTestWithClientFactory(t, testAutoConfDefaultConfig, &initialEvent, makeFactory, + func(p autoConfTestParams) { + defer releaseGate() + + p.awaitClient() + p.awaitClient() + env1 := p.awaitEnvironment(testAutoConfEnv1.id) + env2 := p.awaitEnvironment(testAutoConfEnv2.id) + + // Rotate env 1. Its re-anchor build blocks inside the factory and stays blocked. + p.stream.Enqueue(makeAutoConfPatchEvent(env1Rotated)) + helpers.RequireValue(t, buildStarted, time.Second*5, + "env 1's re-anchor build never started") + + // Rotate env 2 while env 1 is still stuck. This is the assertion that matters: before + // per-environment serialization this patch could not even begin to be processed until env + // 1's build returned or timed out. + p.stream.Enqueue(makeAutoConfPatchEvent(env2Rotated)) + p.awaitCredentialsUpdated(env2, env2Rotated.params()) + + // Env 2 completed its rotation and env 1 did not: exactly one new client was built, and it + // is env 2's. Env 1 is isolated, not skipped or reordered. + rotatedClient := p.awaitClient() + require.Equal(t, env2Rotated.SDKKey(), rotatedClient.Key, + "the only completed rotation should be env 2's") + p.shouldNotCreateClient(100 * time.Millisecond) + + // Releasing the build lets env 1 finish its own rotation. + releaseGate() + env1Client := p.awaitClient() + require.Equal(t, env1Rotated.SDKKey(), env1Client.Key) + p.awaitCredentialsUpdated(env1, env1Rotated.params()) + }) +}