feat(experiments): configurable analysis baseline variant

frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx

@@ -26,6 +26,7 @@ import { experimentLogic } from '~/scenes/experiments/experimentLogic'
 import { modalsLogic } from '~/scenes/experiments/modalsLogic'
 import { MultivariateFlagVariant } from '~/types'
 
+import { resolveBaselineVariantKey } from '../utils'
 import { HoldoutSelector } from './HoldoutSelector'
 import { VariantScreenshot } from './VariantScreenshot'
 import { VariantTag } from './VariantTag'
@@ -114,11 +115,14 @@ export function DistributionTable(): JSX.Element {
     const excludedVariantsEnabled = useFeatureFlag('EXPERIMENTS_EXCLUDED_VARIANTS')
 
     /**
-     * This is future-proofing to match the experiment query runner backend, that uses
-     * the baseline variant key to determine the baseline variant.
+     * This matches the experiment query runner backend, which uses the baseline variant
+     * key (falling back to control, then the first variant) to determine the baseline.
      */
-    const baselineKey = experiment.stats_config?.baseline_variant_key || 'control'
-    const variants = experiment.feature_flag?.filters.multivariate?.variants || []
+    const variants: MultivariateFlagVariant[] = experiment.feature_flag?.filters.multivariate?.variants || []
+    const baselineKey = resolveBaselineVariantKey(
+        variants.map((v) => v.key),
+        experiment.stats_config?.baseline_variant_key
+    )
 
     /**
      * We use this check to disable the toggle if there's only one test variant left.
@@ -265,7 +269,7 @@ export function DistributionTable(): JSX.Element {
           ]
         : []
 
-    const variantData = (experiment.feature_flag?.filters.multivariate?.variants || []).map((variant) => ({
+    const variantData = variants.map((variant) => ({
         ...variant,
         rollout_percentage:
             variant.rollout_percentage * ((100 - (experiment.holdout?.filters[0].rollout_percentage || 0)) / 100),

frontend/src/scenes/experiments/ExperimentView/SettingsTab.tsx

@@ -7,18 +7,19 @@ import { LinkedHogFunctions } from 'scenes/hog-functions/list/LinkedHogFunctions
 import { experimentsConfigLogic } from 'scenes/settings/environment/experimentsConfigLogic'
 import { urls } from 'scenes/urls'
 
-import { ExperimentStatsMethod, PropertyFilterType, PropertyOperator } from '~/types'
+import { ExperimentStatsMethod, MultivariateFlagVariant, PropertyFilterType, PropertyOperator } from '~/types'
 
 import { DEFAULT_LOOKBACK_DAYS } from '../constants'
 import { experimentLogic } from '../experimentLogic'
 import { modalsLogic } from '../modalsLogic'
+import { resolveBaselineVariantKey } from '../utils'
 import { getCupedSelection, resolveCupedEnabled, resolveCupedLookbackDays } from './cuped'
 import { CupedModal } from './CupedModal'
 import { resolveSequentialEnabled } from './sequential'
 import { StatsMethodModal } from './StatsMethodModal'
 
 export function SettingsTab(): JSX.Element {
-    const { experiment, statsMethod, variants } = useValues(experimentLogic)
+    const { experiment, statsMethod, excludedVariants, experimentUpdateLoading } = useValues(experimentLogic)
     const { updateExperimentSettings } = useActions(experimentLogic)
     const { openStatsEngineModal, openCupedModal } = useActions(modalsLogic)
     const { experimentsConfig } = useValues(experimentsConfigLogic)
@@ -50,6 +51,15 @@ export function SettingsTab(): JSX.Element {
     // Only show alerts section for saved experiments, as the alert relies on experiment.id for filtering
     const shouldShowSignificanceAlerts = typeof experiment.id === 'number'
 
+    const experimentVariants = (
+        (experiment.feature_flag?.filters?.multivariate?.variants ?? []) as MultivariateFlagVariant[]
+    ).filter((variant) => !excludedVariants.includes(variant.key))
+    const variantKeys = experimentVariants.map((v) => v.key)
+    const configuredBaselineKey = experiment.stats_config?.baseline_variant_key
+    const effectiveBaselineKey = resolveBaselineVariantKey(variantKeys, configuredBaselineKey)
+    // The stored baseline can point at a variant that was since removed from the flag.
+    const baselineMissing = !!configuredBaselineKey && !variantKeys.includes(configuredBaselineKey)
+
     return (
         <div className="flex flex-col gap-8">
             <div>
@@ -90,17 +100,28 @@ export function SettingsTab(): JSX.Element {
             <div>
                 <h2 className="font-semibold text-lg">Baseline variant</h2>
                 <LemonSelect
-                    value={experiment.stats_config?.baseline_variant_key ?? 'control'}
-                    options={variants.map((v) => ({
+                    value={effectiveBaselineKey}
+                    options={experimentVariants.map((v) => ({
                         value: v.key,
                         label: v.key,
                     }))}
-                    onChange={(value) => {
+                    loading={experimentUpdateLoading}
+                    disabledReason={experimentUpdateLoading ? 'Saving baseline\u2026' : undefined}
+                    onSelect={(value) => {
+                        if (value === configuredBaselineKey) {
+                            return
+                        }
                         updateExperimentSettings({
                             stats_config: { ...experiment.stats_config, baseline_variant_key: value },
                         })
                     }}
                 />
+                {baselineMissing && (
+                    <p className="text-warning text-xs mt-1">
+                        The previously selected baseline “{configuredBaselineKey}” is no longer a variant; analysis
+                        falls back to “{effectiveBaselineKey}” until you choose another.
+                    </p>
+                )}
                 <p className="text-muted text-xs mt-1">The variant all others are compared against.</p>
             </div>
             <div>

frontend/src/scenes/experiments/ExperimentWizard/steps/AnalyticsStep.tsx

@@ -1,15 +1,30 @@
 import { useActions, useValues } from 'kea'
 
+import { LemonSelect } from '@posthog/lemon-ui'
+
 import { LemonBanner } from 'lib/lemon-ui/LemonBanner'
 
+import { MultivariateFlagVariant } from '~/types'
+
 import { ExposureCriteriaPanel } from '../../ExperimentForm/ExposureCriteriaPanel'
 import { MetricsPanel } from '../../ExperimentForm/MetricsPanel'
+import { resolveBaselineVariantKey } from '../../utils'
 import { experimentWizardLogic } from '../experimentWizardLogic'
 
 export function AnalyticsStep(): JSX.Element {
     const { experiment, sharedMetrics } = useValues(experimentWizardLogic)
     const { setExperiment, setExposureCriteria, setSharedMetrics } = useActions(experimentWizardLogic)
 
+    const baselineVariants = (experiment.parameters?.feature_flag_variants ?? []) as MultivariateFlagVariant[]
+    const baselineVariantKeys = baselineVariants.map((v) => v.key)
+    const effectiveBaselineKey = resolveBaselineVariantKey(
+        baselineVariantKeys,
+        experiment.stats_config?.baseline_variant_key
+    )
+    const baselineSelectValue = baselineVariantKeys.includes(experiment.stats_config?.baseline_variant_key ?? '')
+        ? experiment.stats_config?.baseline_variant_key
+        : undefined
+
     return (
         <div className="space-y-6">
             <div className="space-y-4">
@@ -72,6 +87,26 @@ export function AnalyticsStep(): JSX.Element {
                 </div>
             </div>
 
+            {baselineVariantKeys.length > 0 && (
+                <div>
+                    <h3 className="text-lg font-semibold mb-1">Which variant is the baseline?</h3>
+                    <p className="text-muted text-sm mb-2">
+                        All other variants are compared against this one. You can change it later in settings.
+                    </p>
+                    <LemonSelect
+                        value={baselineSelectValue}
+                        placeholder={effectiveBaselineKey}
+                        options={baselineVariants.map((v) => ({ value: v.key, label: v.key }))}
+                        onSelect={(value) =>
+                            setExperiment({
+                                ...experiment,
+                                stats_config: { ...experiment.stats_config, baseline_variant_key: value },
+                            })
+                        }
+                    />
+                </div>
+            )}
+
             <LemonBanner type="info">
                 You can always refine your analytics configuration and metrics after saving.
             </LemonBanner>

frontend/src/scenes/experiments/utils.test.ts

@@ -40,6 +40,7 @@ import {
     isLegacyExperiment,
     isLegacyExperimentQuery,
     percentageDistribution,
+    resolveBaselineVariantKey,
 } from './utils'
 
 describe('utils', () => {
@@ -1383,3 +1384,28 @@ describe('getEventCountQuery', () => {
         expect(query).toBeNull()
     })
 })
+
+describe('resolveBaselineVariantKey', () => {
+    it('returns the configured key when present among the variants', () => {
+        expect(resolveBaselineVariantKey(['control', 'test'], 'test')).toBe('test')
+        expect(resolveBaselineVariantKey(['control', 'test'], 'control')).toBe('control')
+    })
+
+    it('falls back to control when the configured key was removed', () => {
+        expect(resolveBaselineVariantKey(['control', 'test-a', 'test-b'], 'removed')).toBe('control')
+    })
+
+    it('falls back to the first variant when neither configured nor control are present', () => {
+        expect(resolveBaselineVariantKey(['variant-a', 'variant-b'], 'removed')).toBe('variant-a')
+    })
+
+    it('defaults to control when nothing is configured', () => {
+        expect(resolveBaselineVariantKey(['control', 'test'])).toBe('control')
+        expect(resolveBaselineVariantKey(['control', 'test'], null)).toBe('control')
+    })
+
+    it('returns control as a safe default when there are no variants', () => {
+        expect(resolveBaselineVariantKey([], 'control')).toBe('control')
+        expect(resolveBaselineVariantKey([])).toBe('control')
+    })
+})

frontend/src/scenes/experiments/utils.ts

@@ -951,3 +951,24 @@ export function getOrderedMetricsWithResults(
             metricIndex: originalIndexMap.get(metric.uuid) ?? index, // Original position for retry
         }))
 }
+
+// The conventional baseline/control variant key. Experiments default their analysis
+// baseline to this when `stats_config.baseline_variant_key` is unset.
+export const DEFAULT_BASELINE_VARIANT_KEY = 'control'
+
+/**
+ * Resolve the effective analysis baseline variant key against the variants actually present.
+ *
+ * Mirrors the backend `resolve_baseline_variant_key`: prefer the configured key, then the
+ * conventional `control` variant, then the first available variant. This keeps the UI in sync
+ * with what the query runner computes when the configured baseline was removed from the flag.
+ */
+export function resolveBaselineVariantKey(variantKeys: string[], configuredKey?: string | null): string {
+    if (configuredKey && variantKeys.includes(configuredKey)) {
+        return configuredKey
+    }
+    if (variantKeys.includes(DEFAULT_BASELINE_VARIANT_KEY)) {
+        return DEFAULT_BASELINE_VARIANT_KEY
+    }
+    return variantKeys[0] ?? DEFAULT_BASELINE_VARIANT_KEY
+}

products/experiments/backend/experiment_service.py

@@ -773,6 +773,7 @@ def create_experiment(
             only_count_matured_users = team_config.default_only_count_matured_users
 
         stats_method = "bayesian" if stats_config is None else stats_config.get("method", "bayesian")
+        baseline_variant_key = None if stats_config is None else stats_config.get("baseline_variant_key")
         if metrics is not None:
             for metric in metrics:
                 metric["fingerprint"] = compute_metric_fingerprint(
@@ -782,6 +783,7 @@ def create_experiment(
                     exposure_criteria,
                     only_count_matured_users=only_count_matured_users,
                     excluded_variants=(parameters or {}).get("excluded_variants"),
+                    baseline_variant_key=baseline_variant_key,
                 )
         if metrics_secondary is not None:
             for metric in metrics_secondary:
@@ -792,6 +794,7 @@ def create_experiment(
                     exposure_criteria,
                     only_count_matured_users=only_count_matured_users,
                     excluded_variants=(parameters or {}).get("excluded_variants"),
+                    baseline_variant_key=baseline_variant_key,
                 )
 
         self.validate_no_duplicate_metric_uuids(metrics, metrics_secondary)
@@ -1085,6 +1088,7 @@ def _recompute_fingerprints(
     ) -> list[dict]:
         """Recompute fingerprints for a list of metrics. Returns a new list with updated fingerprints."""
         stats_method = "bayesian" if stats_config is None else stats_config.get("method", "bayesian")
+        baseline_variant_key = None if stats_config is None else stats_config.get("baseline_variant_key")
         updated = []
         for metric in metrics:
             metric_copy = deepcopy(metric)
@@ -1095,6 +1099,7 @@ def _recompute_fingerprints(
                 exposure_criteria,
                 only_count_matured_users=only_count_matured_users,
                 excluded_variants=excluded_variants,
+                baseline_variant_key=baseline_variant_key,
             )
             updated.append(metric_copy)
         return updated

products/experiments/backend/hogql_queries/experiment_metric_fingerprint.py

@@ -29,6 +29,7 @@ def compute_metric_fingerprint(
     exposure_criteria: dict | None = None,
     only_count_matured_users: bool = False,
     excluded_variants: list[str] | None = None,
+    baseline_variant_key: str | None = None,
 ) -> str:
     """
     Compute fingerprint for a metric.
@@ -41,6 +42,8 @@ def compute_metric_fingerprint(
         only_count_matured_users
         excluded_variants: Variant keys excluded from analysis — changing the set
             invalidates cached results since it alters which data is computed
+        baseline_variant_key: Variant key used as the analysis baseline — changing it
+            invalidates cached results since it alters each comparison result
 
     Returns:
         SHA256 hash string representing the metric fingerprint
@@ -74,6 +77,9 @@ def compute_metric_fingerprint(
     if excluded_variants:
         fingerprint_data["excluded_variants"] = sorted(set(excluded_variants))
 
+    if baseline_variant_key:
+        fingerprint_data["baseline_variant_key"] = baseline_variant_key
+
     # Create deterministic JSON string with sorted keys at all levels
     json_str = json.dumps(fingerprint_data, sort_keys=True, separators=(",", ":"))
 

products/experiments/backend/hogql_queries/experiment_query_runner.py

@@ -59,6 +59,7 @@
     get_experiment_stats_method,
     get_frequentist_experiment_result,
     get_variant_results,
+    resolve_baseline_variant_key,
     split_baseline_and_test_variants,
 )
 from products.experiments.backend.metric_utils import get_default_metric_title
@@ -505,7 +506,11 @@ def _has_breakdown(self, variant_results: list[tuple[tuple[str, ...] | None, Exp
 
     def _calculate_statistics_for_variants(self, variants: list[ExperimentStatsBase]) -> ExperimentQueryResponse:
         """Calculate statistical analysis results for a set of variants."""
-        control_variant, test_variants = split_baseline_and_test_variants(variants, self.baseline_variant_key)
+        # Resolve against the variants actually present so a baseline that was removed from the
+        # flag (bypassing experiment-update validation) degrades to control/first instead of
+        # erroring the whole scorecard.
+        baseline_key = resolve_baseline_variant_key([variant.key for variant in variants], self.baseline_variant_key)
+        control_variant, test_variants = split_baseline_and_test_variants(variants, baseline_key)
 
         if self.stats_method == "frequentist":
             return get_frequentist_experiment_result(

products/experiments/backend/hogql_queries/test/experiment_query_runner/test_excluded_variants.py

@@ -98,3 +98,16 @@ def test_fingerprint_changes_when_excluded_variants_change():
     assert fp_two != fp_one
     assert fp_two_reversed == fp_two, "Order of excluded keys must not affect fingerprint"
     assert fp_one != fp_one_other
+
+
+def test_fingerprint_changes_when_baseline_variant_changes():
+    metric = {"kind": "ExperimentMeanMetric", "source": {"kind": "EventsNode", "event": "$pageview"}}
+    start = "2026-01-01T00:00:00+00:00"
+
+    fp_default = compute_metric_fingerprint(metric, start, baseline_variant_key=None)
+    fp_control = compute_metric_fingerprint(metric, start, baseline_variant_key="control")
+    fp_test = compute_metric_fingerprint(metric, start, baseline_variant_key="test")
+
+    assert fp_default == compute_metric_fingerprint(metric, start)
+    assert fp_control != fp_default
+    assert fp_test != fp_control

products/experiments/backend/hogql_queries/test/test_stats_config.py

@@ -18,6 +18,7 @@
 from products.experiments.backend.hogql_queries.utils import (
     get_bayesian_experiment_result,
     get_frequentist_experiment_result,
+    resolve_baseline_variant_key,
     split_baseline_and_test_variants,
 )
 from products.experiments.backend.models.experiment import Experiment
@@ -400,6 +401,41 @@ def test_experiment_query_runner_reads_baseline_from_stats_config(self, _name, s
 
         self.assertEqual(runner.baseline_variant_key, expected_baseline)
 
+    @parameterized.expand(
+        [
+            # configured baseline present -> used as-is
+            ("configured_present", ["control", "test"], "test", "test"),
+            ("configured_present_control", ["control", "test"], "control", "control"),
+            # configured baseline missing, control present -> falls back to control
+            ("missing_falls_back_to_control", ["control", "test-a", "test-b"], "removed", "control"),
+            # configured baseline missing and no control -> falls back to first variant
+            ("missing_no_control_first_variant", ["variant-a", "variant-b"], "removed", "variant-a"),
+            # no variants at all -> returns the configured key unchanged (degenerate)
+            ("no_variants", [], "control", "control"),
+            # default (None configured) resolves to control when present
+            ("none_configured_control_present", ["control", "test"], None, "control"),
+        ]
+    )
+    def test_resolve_baseline_variant_key(self, _name, variant_keys, configured, expected):
+        if configured is None:
+            self.assertEqual(resolve_baseline_variant_key(variant_keys), expected)
+        else:
+            self.assertEqual(resolve_baseline_variant_key(variant_keys, configured), expected)
+
+    def test_resolve_then_split_recovers_when_baseline_removed(self):
+        # When the configured baseline variant was removed from the flag, resolving first
+        # keeps split_baseline_and_test_variants working instead of raising "No control variant".
+        variants = [
+            self.create_variant("control", sum_val=100.0, sum_squares=10500.0, samples=1000),
+            self.create_variant("test", sum_val=120.0, sum_squares=14500.0, samples=1000),
+        ]
+        resolved = resolve_baseline_variant_key([v.key for v in variants], "removed-baseline")
+        self.assertEqual(resolved, "control")
+
+        baseline, test_variants = split_baseline_and_test_variants(variants, resolved)
+        self.assertEqual(baseline.key, "control")
+        self.assertEqual([v.key for v in test_variants], ["test"])
+
 
 class TestSequentialStatsConfig(APIBaseTest):
     """Verify sequential_testing flags propagate from stats_config (and team defaults) into the engine."""