fix(autoconfig): process each environment on its own queue

.github/variables/go-versions.env

@@ -1,2 +1,2 @@
-latest=1.26.4
-penultimate=1.25.11
+latest=1.26.6
+penultimate=1.25.13

Dockerfile

@@ -2,7 +2,7 @@
 
 # This is a standalone Dockerfile that does not depend on goreleaser building the binary
 # It is NOT the version that is pushed to dockerhub
-FROM golang:1.26.4-alpine3.24 as builder
+FROM golang:1.26.6-alpine3.24 as builder
 # See "Runtime platform versions" in CONTRIBUTING.md
 
 RUN apk --no-cache add \

config/config.go

@@ -38,6 +38,15 @@ const (
 	// DefaultEventCapacity is the default value for EventsConfig.Capacity if not specified.
 	DefaultEventCapacity = 1000
 
+	// DefaultMetricsCapacity is the default for EventsConfig.MetricsCapacity: the maximum queue size
+	// for the usage-metrics publisher, which emits one event per unique connection on each flush.
+	// It exceeds DefaultEventCapacity because high-concurrency nodes exceed 1000 connections.
+	DefaultMetricsCapacity = 10000
+
+	// DefaultMetricsInitialCapacity is the number of events the usage-metrics publisher queue
+	// preallocates space for. The queue grows on demand from this size up to MetricsCapacity.
+	DefaultMetricsInitialCapacity = 1000
+
 	// DefaultHeartbeatInterval is the default value for MainConfig.HeartBeatInterval if not specified.
 	DefaultHeartbeatInterval = time.Minute * 3
 
@@ -95,6 +104,9 @@ const (
 	// credentials to be revoked nearly instantaneously. It is not necessarily a recommendation.
 	// It likely doesn't make sense to use an interval this frequent in production use-cases.
 	minimumCredentialCleanupInterval = 100 * time.Millisecond
+	// minimumMetricsCapacity is the smallest value accepted for EventsConfig.MetricsCapacity. Usage
+	// metrics report account usage, so the floor is the historical default of 1000.
+	minimumMetricsCapacity = 1000
 )
 
 // DefaultLoggers is the default logging configuration used by Relay.
@@ -153,6 +165,7 @@ type MainConfig struct {
 	GracefulShutdownTimeout          ct.OptDuration           `conf:"GRACEFUL_SHUTDOWN_TIMEOUT"`
 	HeartbeatInterval                ct.OptDuration           `conf:"HEARTBEAT_INTERVAL"`
 	MaxClientConnectionTime          ct.OptDuration           `conf:"MAX_CLIENT_CONNECTION_TIME"`
+	MaxClientRequestBodySize         ct.OptBase2Bytes         `conf:"MAX_CLIENT_REQUEST_BODY_SIZE"`
 	PingStreamJitterTime             ct.OptDuration           `conf:"PING_STREAM_JITTER_TIME"`
 	DisconnectedStatusTime           ct.OptDuration           `conf:"DISCONNECTED_STATUS_TIME"`
 	TLSEnabled                       bool                     `conf:"TLS_ENABLED"`
@@ -196,6 +209,7 @@ type EventsConfig struct {
 	SendEvents            bool                     `conf:"USE_EVENTS"`
 	FlushInterval         ct.OptDuration           `conf:"EVENTS_FLUSH_INTERVAL"`
 	Capacity              ct.OptIntGreaterThanZero `conf:"EVENTS_CAPACITY"`
+	MetricsCapacity       ct.OptIntGreaterThanZero `conf:"EVENTS_METRICS_CAPACITY"`
 	InlineUsers           bool                     `conf:"EVENTS_INLINE_USERS"`
 	MaxInboundPayloadSize ct.OptBase2Bytes         `conf:"EVENTS_MAX_INBOUND_PAYLOAD_SIZE"`
 }

config/config_validation.go

@@ -17,6 +17,7 @@ var (
 	errOfflineModePropertiesWithNoFile = errors.New("must specify offline mode filename if other offline mode properties are set")
 	errOfflineModeWithEnvironments     = errors.New("cannot configure specific environments if offline mode is enabled")
 	errMaxInboundPayloadSize           = errors.New("max inbound payload size must be greater than zero")
+	errMaxClientRequestBodySize        = errors.New("max client request body size must be greater than zero")
 	errAutoConfWithoutDBDisambig       = errors.New(`when using auto-configuration with database storage, database prefix (or,` +
 		` if using DynamoDB, table name) must be specified and must contain "` + AutoConfigEnvironmentIDPlaceholder + `"`)
 	errRedisURLWithHostAndPort                 = errors.New("please specify Redis URL or host/port, but not both")
@@ -30,6 +31,8 @@ var (
 	errInvalidCredentialCleanupInterval        = fmt.Errorf("expired credential cleanup interval must be >= %s", minimumCredentialCleanupInterval)
 )
 
+const warnMetricsCapacityBelowMinimum = "configured usage metrics event capacity of %d is below the minimum of %d; using %[2]d instead"
+
 func errEnvironmentWithNoSDKKey(envName string) error {
 	return fmt.Errorf("SDK key is required for environment %q", envName)
 }
@@ -85,6 +88,8 @@ func ValidateConfig(c *Config, loggers ldlog.Loggers) error {
 	validateOfflineMode(&result, c)
 	validateCredentialCleanupInterval(&result, c)
 	validateMaxInboundPayloadSize(&result, c)
+	validateMaxClientRequestBodySize(&result, c)
+	validateMetricsCapacity(c, loggers)
 
 	return result.GetError()
 }
@@ -234,6 +239,29 @@ func validateMaxInboundPayloadSize(result *ct.ValidationResult, c *Config) {
 	}
 }
 
+func validateMaxClientRequestBodySize(result *ct.ValidationResult, c *Config) {
+	if c.Main.MaxClientRequestBodySize.IsDefined() {
+		size := c.Main.MaxClientRequestBodySize.GetOrElse(0)
+		if size <= 0 {
+			result.AddError(nil, errMaxClientRequestBodySize)
+		}
+	}
+}
+
+// validateMetricsCapacity enforces the minimum queue capacity for the usage-metrics event publisher.
+// Rather than fail startup on a too-small value, it clamps the value up to the minimum and warns.
+func validateMetricsCapacity(c *Config, loggers ldlog.Loggers) {
+	if !c.Events.MetricsCapacity.IsDefined() {
+		return
+	}
+	if c.Events.MetricsCapacity.GetOrElse(0) < minimumMetricsCapacity {
+		loggers.Warnf(warnMetricsCapacityBelowMinimum, c.Events.MetricsCapacity.GetOrElse(0), minimumMetricsCapacity)
+		// This value is a constant known to be greater than zero, so the constructor cannot fail.
+		clamped, _ := ct.NewOptIntGreaterThanZero(minimumMetricsCapacity)
+		c.Events.MetricsCapacity = clamped
+	}
+}
+
 func validateConfigDatabases(result *ct.ValidationResult, c *Config, loggers ldlog.Loggers) {
 	normalizeRedisConfig(result, c)
 

config/config_validation_test.go

@@ -0,0 +1,39 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/launchdarkly/go-sdk-common/v3/ldlog"
+	"github.com/launchdarkly/go-sdk-common/v3/ldlogtest"
+)
+
+func TestValidateMetricsCapacity(t *testing.T) {
+	t.Run("unset value is left undefined", func(t *testing.T) {
+		var c Config
+		mockLog := ldlogtest.NewMockLog()
+		require.NoError(t, ValidateConfig(&c, mockLog.Loggers))
+		assert.False(t, c.Events.MetricsCapacity.IsDefined())
+		assert.Len(t, mockLog.GetOutput(ldlog.Warn), 0)
+	})
+
+	t.Run("value at or above the minimum is left unchanged", func(t *testing.T) {
+		var c Config
+		c.Events.MetricsCapacity = mustOptIntGreaterThanZero(2000)
+		mockLog := ldlogtest.NewMockLog()
+		require.NoError(t, ValidateConfig(&c, mockLog.Loggers))
+		assert.Equal(t, 2000, c.Events.MetricsCapacity.GetOrElse(0))
+		assert.Len(t, mockLog.GetOutput(ldlog.Warn), 0)
+	})
+
+	t.Run("value below the minimum is clamped up and warns", func(t *testing.T) {
+		var c Config
+		c.Events.MetricsCapacity = mustOptIntGreaterThanZero(500)
+		mockLog := ldlogtest.NewMockLog()
+		require.NoError(t, ValidateConfig(&c, mockLog.Loggers))
+		assert.Equal(t, minimumMetricsCapacity, c.Events.MetricsCapacity.GetOrElse(0))
+		mockLog.AssertMessageMatch(t, true, ldlog.Warn, "usage metrics event capacity of 500 is below the minimum of 1000")
+	})
+}

config/test_data_configs_invalid_test.go

@@ -43,9 +43,21 @@ func makeInvalidConfigs() []testDataInvalidConfig {
 		makeInvalidConfigDynamoDBNoPrefixOrTableName(),
 		makeInvalidConfigDynamoDBAutoConfNoPrefixOrTableName(),
 		makeInvalidConfigMultipleDatabases(),
+		makeInvalidConfigMaxClientRequestBodySize("0B"),
 	}
 }
 
+func makeInvalidConfigMaxClientRequestBodySize(size string) testDataInvalidConfig {
+	c := testDataInvalidConfig{name: "max client request body size " + size}
+	c.envVarsError = errMaxClientRequestBodySize.Error()
+	c.envVars = map[string]string{"MAX_CLIENT_REQUEST_BODY_SIZE": size}
+	c.fileContent = `
+[Main]
+MaxClientRequestBodySize = ` + size + `
+`
+	return c
+}
+
 func makeInvalidConfigMissingSDKKey() testDataInvalidConfig {
 	c := testDataInvalidConfig{name: "environment without SDK key"}
 	c.fileContent = `

config/test_data_configs_valid_test.go

@@ -49,6 +49,14 @@ func mustOptIntGreaterThanZero(n int) ct.OptIntGreaterThanZero {
 	return o
 }
 
+func mustOptBase2Bytes(s string) ct.OptBase2Bytes {
+	o, err := ct.NewOptBase2BytesFromString(s)
+	if err != nil {
+		panic(err)
+	}
+	return o
+}
+
 func newOptURLAbsoluteMustBeValid(urlString string) ct.OptURLAbsolute {
 	o, err := ct.NewOptURLAbsoluteFromString(urlString)
 	if err != nil {
@@ -110,6 +118,7 @@ func makeValidConfigAllBaseProperties() testDataValidConfig {
 			IgnoreConnectionErrors:           true,
 			HeartbeatInterval:                ct.NewOptDuration(90 * time.Second),
 			MaxClientConnectionTime:          ct.NewOptDuration(30 * time.Minute),
+			MaxClientRequestBodySize:         mustOptBase2Bytes("5MiB"),
 			DisconnectedStatusTime:           ct.NewOptDuration(3 * time.Minute),
 			TLSEnabled:                       true,
 			TLSCert:                          "cert",
@@ -126,6 +135,7 @@ func makeValidConfigAllBaseProperties() testDataValidConfig {
 			EventsURI:             newOptURLAbsoluteMustBeValid("http://events"),
 			FlushInterval:         ct.NewOptDuration(120 * time.Second),
 			Capacity:              mustOptIntGreaterThanZero(500),
+			MetricsCapacity:       mustOptIntGreaterThanZero(50000),
 			InlineUsers:           true,
 			MaxInboundPayloadSize: ct.OptBase2Bytes{},
 		}
@@ -161,6 +171,7 @@ func makeValidConfigAllBaseProperties() testDataValidConfig {
 		"IGNORE_CONNECTION_ERRORS":            "1",
 		"HEARTBEAT_INTERVAL":                  "90s",
 		"MAX_CLIENT_CONNECTION_TIME":          "30m",
+		"MAX_CLIENT_REQUEST_BODY_SIZE":        "5MiB",
 		"DISCONNECTED_STATUS_TIME":            "3m",
 		"TLS_ENABLED":                         "1",
 		"TLS_CERT":                            "cert",
@@ -173,6 +184,7 @@ func makeValidConfigAllBaseProperties() testDataValidConfig {
 		"EVENTS_HOST":                         "http://events",
 		"EVENTS_FLUSH_INTERVAL":               "120s",
 		"EVENTS_CAPACITY":                     "500",
+		"EVENTS_METRICS_CAPACITY":             "50000",
 		"EVENTS_INLINE_USERS":                 "1",
 		"LD_ENV_earth":                        "earth-sdk",
 		"LD_MOBILE_KEY_earth":                 "earth-mob",
@@ -203,6 +215,7 @@ ExitAlways = 1
 IgnoreConnectionErrors = 1
 HeartbeatInterval = 90s
 MaxClientConnectionTime = 30m
+MaxClientRequestBodySize = "5MiB"
 PingStreamJitterTime = 5m
 DisconnectedStatusTime = 3m
 TLSEnabled = 1
@@ -219,6 +232,7 @@ SendEvents = 1
 EventsUri = "http://events"
 FlushInterval = 120s
 Capacity = 500
+MetricsCapacity = 50000
 InlineUsers = 1
 
 [Environment "earth"]

docs/configuration.md

@@ -58,6 +58,7 @@ For **Duration** settings, the value should be be an integer followed by `ms`, `
 | `gracefulShutdownTimeout`          | `GRACEFUL_SHUTDOWN_TIMEOUT`           | Duration | `30s`   | How long the Relay Proxy should wait for active connections to complete before forcefully shutting down when receiving a termination signal. This allows for graceful shutdown of the server, ensuring that in-flight requests are completed. The value should be a duration string like `30s` or `1m`.                                                                                                                                                                              |
 | `heartbeatInterval`                | `HEARTBEAT_INTERVAL`                  |  Number  | `3m`    | Interval for heartbeat messages to prevent read timeouts on streaming connections. Assumed to be in seconds if no unit is specified.                                                                                                                                                                                                                                                                                                                                               |
 | `maxClientConnectionTime`          | `MAX_CLIENT_CONNECTION_TIME`          | Duration | none    | Maximum amount of time that Relay will allow a streaming connection from an SDK client to remain open. _(3)_                                                                                                                                                                                                                                                                                                                                                                       |
+| `maxClientRequestBodySize`         | `MAX_CLIENT_REQUEST_BODY_SIZE`        |   Unit   | none    | Maximum size of a `REPORT` request body that Relay will read when evaluating flags for a client-side, mobile, or server-side SDK. _(10)_                                                                                                                                                                                                                                                                                                                                             |
 | `disconnectedStatusTime`           | `DISCONNECTED_STATUS_TIME`            | Duration | `1m`    | How long a stream connection can be interrupted before Relay reports the status as "disconnected." _(4)_                                                                                                                                                                                                                                                                                                                                                                           |
 | `tlsEnabled`                       | `TLS_ENABLED`                         | Boolean  | `false` | Enable TLS on the Relay Proxy. Read: [Using TLS](./tls.md).                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `tlsCert`                          | `TLS_CERT`                            |  String  |         | Required if `tlsEnabled` is true. Path to TLS certificate file.                                                                                                                                                                                                                                                                                                                                                                                                                    |
@@ -126,11 +127,15 @@ To learn more, read [Forwarding events](./events.md).
 | `eventsUri`                   | `EVENTS_HOST`                        |   URI    | _(7)_   | URI for the LaunchDarkly events service                                                                                                                                                                                   |
 | `flushInterval`               | `EVENTS_FLUSH_INTERVAL`              | Duration | `5s`    | Controls how long the SDK buffers events before sending them back to our server. If your server generates many events per second, we suggest decreasing the flush interval and/or increasing capacity to meet your needs. |
 | `capacity`                    | `EVENTS_CAPACITY`                    |  Number  | `1000`  | Maximum number of events to accumulate for each flush interval.                                                                                                                                                           |
+| `metricsCapacity`             | `EVENTS_METRICS_CAPACITY`            |  Number  | `10000` | Queue capacity for the usage metrics event publisher, which reports connection usage to LaunchDarkly independently of `capacity`. See note _(9)_.                                                                          |
 | `inlineUsers`                 | `EVENTS_INLINE_USERS`                | Boolean  | `false` | When enabled, individual events (if full event tracking is enabled for the feature flag) will contain all non-private user attributes.                                                                                    |
 | `maxInboundPayloadSize`       | `EVENTS_MAX_INBOUND_PAYLOAD_SIZE`    | Unit     | _(8)_   | Maximum size of an event payload the Relay Proxy will accept from an SDK.                                                                                                                                                 |
 
 _(7)_ See note _(1)_ above. The default value for `eventsUri` is `https://events.launchdarkly.com`.
 _(8)_ The `maxInboundPayloadSize` setting is used to limit the size of the payload that the Relay Proxy will accept from an SDK. This is an optional safety feature to prevent the Relay Proxy from being overwhelmed by a very large payload. The default value is `0B` which provides no restriction on the payload size. The value should be a number followed by a unit: `B` for bytes, `KiB` for kibibytes, `MiB` for mebibytes, `GiB` for gibibytes, `TiB` for tebibytes, `PiB` for pebibytes, or `EiB` for exbibytes. For example, `100MiB` is 100 mebibytes.
+_(9)_ The `metricsCapacity` setting controls the queue for usage metrics events, which report connection usage back to LaunchDarkly and are separate from the analytics events governed by `capacity`. The Relay Proxy emits one usage metrics event per concurrent unique connection on each flush, so this should be set to at least the number of concurrent unique connections you expect a single node to serve. This is the maximum capacity: the default is `10000` and the minimum is `1000` (smaller values are clamped up to `1000` with a warning). The queue is an in-memory buffer held per environment that starts small and grows on demand up to this maximum, so its memory footprint tracks the number of concurrent connections actually served rather than the configured maximum.
+
+_(10)_ The optional `maxClientRequestBodySize` setting limits how much of a `REPORT` evaluation request body the Relay Proxy will read into memory before decoding the context, protecting the process from memory exhaustion caused by oversized request bodies. It applies to the `evalx` context/user endpoints for client-side, mobile, and server-side SDKs. By default it is unset, meaning there is no limit (preserving existing behavior). When set, requests whose body exceeds the limit receive an HTTP `413 Request Entity Too Large` response. The value uses the same units as `maxInboundPayloadSize` (for example, `5MiB`).
 
 
 ### File section: `[Environment "NAME"]`