diff --git a/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx b/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx index 3968b808a427..c02407f49d3b 100644 --- a/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx +++ b/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx @@ -26,6 +26,7 @@ import { experimentLogic } from '~/scenes/experiments/experimentLogic' import { modalsLogic } from '~/scenes/experiments/modalsLogic' import { MultivariateFlagVariant } from '~/types' +import { resolveBaselineVariantKey } from '../utils' import { HoldoutSelector } from './HoldoutSelector' import { VariantScreenshot } from './VariantScreenshot' import { VariantTag } from './VariantTag' @@ -114,11 +115,14 @@ export function DistributionTable(): JSX.Element { const excludedVariantsEnabled = useFeatureFlag('EXPERIMENTS_EXCLUDED_VARIANTS') /** - * This is future-proofing to match the experiment query runner backend, that uses - * the baseline variant key to determine the baseline variant. + * This matches the experiment query runner backend, which uses the baseline variant + * key (falling back to control, then the first variant) to determine the baseline. */ - const baselineKey = experiment.stats_config?.baseline_variant_key || 'control' - const variants = experiment.feature_flag?.filters.multivariate?.variants || [] + const variants: MultivariateFlagVariant[] = experiment.feature_flag?.filters.multivariate?.variants || [] + const baselineKey = resolveBaselineVariantKey( + variants.map((v) => v.key), + experiment.stats_config?.baseline_variant_key + ) /** * We use this check to disable the toggle if there's only one test variant left. @@ -265,7 +269,7 @@ export function DistributionTable(): JSX.Element { ] : [] - const variantData = (experiment.feature_flag?.filters.multivariate?.variants || []).map((variant) => ({ + const variantData = variants.map((variant) => ({ ...variant, rollout_percentage: variant.rollout_percentage * ((100 - (experiment.holdout?.filters[0].rollout_percentage || 0)) / 100), diff --git a/frontend/src/scenes/experiments/ExperimentView/SettingsTab.tsx b/frontend/src/scenes/experiments/ExperimentView/SettingsTab.tsx index 314c1e091c8f..cd8f1273e21e 100644 --- a/frontend/src/scenes/experiments/ExperimentView/SettingsTab.tsx +++ b/frontend/src/scenes/experiments/ExperimentView/SettingsTab.tsx @@ -7,18 +7,19 @@ import { LinkedHogFunctions } from 'scenes/hog-functions/list/LinkedHogFunctions import { experimentsConfigLogic } from 'scenes/settings/environment/experimentsConfigLogic' import { urls } from 'scenes/urls' -import { ExperimentStatsMethod, PropertyFilterType, PropertyOperator } from '~/types' +import { ExperimentStatsMethod, MultivariateFlagVariant, PropertyFilterType, PropertyOperator } from '~/types' import { DEFAULT_LOOKBACK_DAYS } from '../constants' import { experimentLogic } from '../experimentLogic' import { modalsLogic } from '../modalsLogic' +import { resolveBaselineVariantKey } from '../utils' import { getCupedSelection, resolveCupedEnabled, resolveCupedLookbackDays } from './cuped' import { CupedModal } from './CupedModal' import { resolveSequentialEnabled } from './sequential' import { StatsMethodModal } from './StatsMethodModal' export function SettingsTab(): JSX.Element { - const { experiment, statsMethod, variants } = useValues(experimentLogic) + const { experiment, statsMethod, excludedVariants, experimentUpdateLoading } = useValues(experimentLogic) const { updateExperimentSettings } = useActions(experimentLogic) const { openStatsEngineModal, openCupedModal } = useActions(modalsLogic) const { experimentsConfig } = useValues(experimentsConfigLogic) @@ -50,6 +51,15 @@ export function SettingsTab(): JSX.Element { // Only show alerts section for saved experiments, as the alert relies on experiment.id for filtering const shouldShowSignificanceAlerts = typeof experiment.id === 'number' + const experimentVariants = ( + (experiment.feature_flag?.filters?.multivariate?.variants ?? []) as MultivariateFlagVariant[] + ).filter((variant) => !excludedVariants.includes(variant.key)) + const variantKeys = experimentVariants.map((v) => v.key) + const configuredBaselineKey = experiment.stats_config?.baseline_variant_key + const effectiveBaselineKey = resolveBaselineVariantKey(variantKeys, configuredBaselineKey) + // The stored baseline can point at a variant that was since removed from the flag. + const baselineMissing = !!configuredBaselineKey && !variantKeys.includes(configuredBaselineKey) + return (
@@ -90,17 +100,28 @@ export function SettingsTab(): JSX.Element {

Baseline variant

({ + value={effectiveBaselineKey} + options={experimentVariants.map((v) => ({ value: v.key, label: v.key, }))} - onChange={(value) => { + loading={experimentUpdateLoading} + disabledReason={experimentUpdateLoading ? 'Saving baseline\u2026' : undefined} + onSelect={(value) => { + if (value === configuredBaselineKey) { + return + } updateExperimentSettings({ stats_config: { ...experiment.stats_config, baseline_variant_key: value }, }) }} /> + {baselineMissing && ( +

+ The previously selected baseline “{configuredBaselineKey}” is no longer a variant; analysis + falls back to “{effectiveBaselineKey}” until you choose another. +

+ )}

The variant all others are compared against.

diff --git a/frontend/src/scenes/experiments/ExperimentWizard/steps/AnalyticsStep.tsx b/frontend/src/scenes/experiments/ExperimentWizard/steps/AnalyticsStep.tsx index 28a01199d1da..688ffd24eee3 100644 --- a/frontend/src/scenes/experiments/ExperimentWizard/steps/AnalyticsStep.tsx +++ b/frontend/src/scenes/experiments/ExperimentWizard/steps/AnalyticsStep.tsx @@ -1,15 +1,30 @@ import { useActions, useValues } from 'kea' +import { LemonSelect } from '@posthog/lemon-ui' + import { LemonBanner } from 'lib/lemon-ui/LemonBanner' +import { MultivariateFlagVariant } from '~/types' + import { ExposureCriteriaPanel } from '../../ExperimentForm/ExposureCriteriaPanel' import { MetricsPanel } from '../../ExperimentForm/MetricsPanel' +import { resolveBaselineVariantKey } from '../../utils' import { experimentWizardLogic } from '../experimentWizardLogic' export function AnalyticsStep(): JSX.Element { const { experiment, sharedMetrics } = useValues(experimentWizardLogic) const { setExperiment, setExposureCriteria, setSharedMetrics } = useActions(experimentWizardLogic) + const baselineVariants = (experiment.parameters?.feature_flag_variants ?? []) as MultivariateFlagVariant[] + const baselineVariantKeys = baselineVariants.map((v) => v.key) + const effectiveBaselineKey = resolveBaselineVariantKey( + baselineVariantKeys, + experiment.stats_config?.baseline_variant_key + ) + const baselineSelectValue = baselineVariantKeys.includes(experiment.stats_config?.baseline_variant_key ?? '') + ? experiment.stats_config?.baseline_variant_key + : undefined + return (
@@ -72,6 +87,26 @@ export function AnalyticsStep(): JSX.Element {
+ {baselineVariantKeys.length > 0 && ( +
+

Which variant is the baseline?

+

+ All other variants are compared against this one. You can change it later in settings. +

+ ({ value: v.key, label: v.key }))} + onSelect={(value) => + setExperiment({ + ...experiment, + stats_config: { ...experiment.stats_config, baseline_variant_key: value }, + }) + } + /> +
+ )} + You can always refine your analytics configuration and metrics after saving. diff --git a/frontend/src/scenes/experiments/utils.test.ts b/frontend/src/scenes/experiments/utils.test.ts index 44cd97a86170..d842f848f04a 100644 --- a/frontend/src/scenes/experiments/utils.test.ts +++ b/frontend/src/scenes/experiments/utils.test.ts @@ -40,6 +40,7 @@ import { isLegacyExperiment, isLegacyExperimentQuery, percentageDistribution, + resolveBaselineVariantKey, } from './utils' describe('utils', () => { @@ -1383,3 +1384,28 @@ describe('getEventCountQuery', () => { expect(query).toBeNull() }) }) + +describe('resolveBaselineVariantKey', () => { + it('returns the configured key when present among the variants', () => { + expect(resolveBaselineVariantKey(['control', 'test'], 'test')).toBe('test') + expect(resolveBaselineVariantKey(['control', 'test'], 'control')).toBe('control') + }) + + it('falls back to control when the configured key was removed', () => { + expect(resolveBaselineVariantKey(['control', 'test-a', 'test-b'], 'removed')).toBe('control') + }) + + it('falls back to the first variant when neither configured nor control are present', () => { + expect(resolveBaselineVariantKey(['variant-a', 'variant-b'], 'removed')).toBe('variant-a') + }) + + it('defaults to control when nothing is configured', () => { + expect(resolveBaselineVariantKey(['control', 'test'])).toBe('control') + expect(resolveBaselineVariantKey(['control', 'test'], null)).toBe('control') + }) + + it('returns control as a safe default when there are no variants', () => { + expect(resolveBaselineVariantKey([], 'control')).toBe('control') + expect(resolveBaselineVariantKey([])).toBe('control') + }) +}) diff --git a/frontend/src/scenes/experiments/utils.ts b/frontend/src/scenes/experiments/utils.ts index a85cf7aa97d5..752de7477a81 100644 --- a/frontend/src/scenes/experiments/utils.ts +++ b/frontend/src/scenes/experiments/utils.ts @@ -951,3 +951,24 @@ export function getOrderedMetricsWithResults( metricIndex: originalIndexMap.get(metric.uuid) ?? index, // Original position for retry })) } + +// The conventional baseline/control variant key. Experiments default their analysis +// baseline to this when `stats_config.baseline_variant_key` is unset. +export const DEFAULT_BASELINE_VARIANT_KEY = 'control' + +/** + * Resolve the effective analysis baseline variant key against the variants actually present. + * + * Mirrors the backend `resolve_baseline_variant_key`: prefer the configured key, then the + * conventional `control` variant, then the first available variant. This keeps the UI in sync + * with what the query runner computes when the configured baseline was removed from the flag. + */ +export function resolveBaselineVariantKey(variantKeys: string[], configuredKey?: string | null): string { + if (configuredKey && variantKeys.includes(configuredKey)) { + return configuredKey + } + if (variantKeys.includes(DEFAULT_BASELINE_VARIANT_KEY)) { + return DEFAULT_BASELINE_VARIANT_KEY + } + return variantKeys[0] ?? DEFAULT_BASELINE_VARIANT_KEY +} diff --git a/products/experiments/backend/experiment_service.py b/products/experiments/backend/experiment_service.py index ff8fe57aaf1c..91b27f889360 100644 --- a/products/experiments/backend/experiment_service.py +++ b/products/experiments/backend/experiment_service.py @@ -773,6 +773,7 @@ def create_experiment( only_count_matured_users = team_config.default_only_count_matured_users stats_method = "bayesian" if stats_config is None else stats_config.get("method", "bayesian") + baseline_variant_key = None if stats_config is None else stats_config.get("baseline_variant_key") if metrics is not None: for metric in metrics: metric["fingerprint"] = compute_metric_fingerprint( @@ -782,6 +783,7 @@ def create_experiment( exposure_criteria, only_count_matured_users=only_count_matured_users, excluded_variants=(parameters or {}).get("excluded_variants"), + baseline_variant_key=baseline_variant_key, ) if metrics_secondary is not None: for metric in metrics_secondary: @@ -792,6 +794,7 @@ def create_experiment( exposure_criteria, only_count_matured_users=only_count_matured_users, excluded_variants=(parameters or {}).get("excluded_variants"), + baseline_variant_key=baseline_variant_key, ) self.validate_no_duplicate_metric_uuids(metrics, metrics_secondary) @@ -1085,6 +1088,7 @@ def _recompute_fingerprints( ) -> list[dict]: """Recompute fingerprints for a list of metrics. Returns a new list with updated fingerprints.""" stats_method = "bayesian" if stats_config is None else stats_config.get("method", "bayesian") + baseline_variant_key = None if stats_config is None else stats_config.get("baseline_variant_key") updated = [] for metric in metrics: metric_copy = deepcopy(metric) @@ -1095,6 +1099,7 @@ def _recompute_fingerprints( exposure_criteria, only_count_matured_users=only_count_matured_users, excluded_variants=excluded_variants, + baseline_variant_key=baseline_variant_key, ) updated.append(metric_copy) return updated diff --git a/products/experiments/backend/hogql_queries/experiment_metric_fingerprint.py b/products/experiments/backend/hogql_queries/experiment_metric_fingerprint.py index 02941cae8cd9..5e58fea35b4f 100644 --- a/products/experiments/backend/hogql_queries/experiment_metric_fingerprint.py +++ b/products/experiments/backend/hogql_queries/experiment_metric_fingerprint.py @@ -29,6 +29,7 @@ def compute_metric_fingerprint( exposure_criteria: dict | None = None, only_count_matured_users: bool = False, excluded_variants: list[str] | None = None, + baseline_variant_key: str | None = None, ) -> str: """ Compute fingerprint for a metric. @@ -41,6 +42,8 @@ def compute_metric_fingerprint( only_count_matured_users excluded_variants: Variant keys excluded from analysis — changing the set invalidates cached results since it alters which data is computed + baseline_variant_key: Variant key used as the analysis baseline — changing it + invalidates cached results since it alters each comparison result Returns: SHA256 hash string representing the metric fingerprint @@ -74,6 +77,9 @@ def compute_metric_fingerprint( if excluded_variants: fingerprint_data["excluded_variants"] = sorted(set(excluded_variants)) + if baseline_variant_key: + fingerprint_data["baseline_variant_key"] = baseline_variant_key + # Create deterministic JSON string with sorted keys at all levels json_str = json.dumps(fingerprint_data, sort_keys=True, separators=(",", ":")) diff --git a/products/experiments/backend/hogql_queries/experiment_query_runner.py b/products/experiments/backend/hogql_queries/experiment_query_runner.py index c3323852fc98..ec6a280a9f69 100644 --- a/products/experiments/backend/hogql_queries/experiment_query_runner.py +++ b/products/experiments/backend/hogql_queries/experiment_query_runner.py @@ -59,6 +59,7 @@ get_experiment_stats_method, get_frequentist_experiment_result, get_variant_results, + resolve_baseline_variant_key, split_baseline_and_test_variants, ) from products.experiments.backend.metric_utils import get_default_metric_title @@ -505,7 +506,11 @@ def _has_breakdown(self, variant_results: list[tuple[tuple[str, ...] | None, Exp def _calculate_statistics_for_variants(self, variants: list[ExperimentStatsBase]) -> ExperimentQueryResponse: """Calculate statistical analysis results for a set of variants.""" - control_variant, test_variants = split_baseline_and_test_variants(variants, self.baseline_variant_key) + # Resolve against the variants actually present so a baseline that was removed from the + # flag (bypassing experiment-update validation) degrades to control/first instead of + # erroring the whole scorecard. + baseline_key = resolve_baseline_variant_key([variant.key for variant in variants], self.baseline_variant_key) + control_variant, test_variants = split_baseline_and_test_variants(variants, baseline_key) if self.stats_method == "frequentist": return get_frequentist_experiment_result( diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_excluded_variants.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_excluded_variants.py index e5dea739f3ff..bd4df712f98b 100644 --- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_excluded_variants.py +++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_excluded_variants.py @@ -98,3 +98,16 @@ def test_fingerprint_changes_when_excluded_variants_change(): assert fp_two != fp_one assert fp_two_reversed == fp_two, "Order of excluded keys must not affect fingerprint" assert fp_one != fp_one_other + + +def test_fingerprint_changes_when_baseline_variant_changes(): + metric = {"kind": "ExperimentMeanMetric", "source": {"kind": "EventsNode", "event": "$pageview"}} + start = "2026-01-01T00:00:00+00:00" + + fp_default = compute_metric_fingerprint(metric, start, baseline_variant_key=None) + fp_control = compute_metric_fingerprint(metric, start, baseline_variant_key="control") + fp_test = compute_metric_fingerprint(metric, start, baseline_variant_key="test") + + assert fp_default == compute_metric_fingerprint(metric, start) + assert fp_control != fp_default + assert fp_test != fp_control diff --git a/products/experiments/backend/hogql_queries/test/test_stats_config.py b/products/experiments/backend/hogql_queries/test/test_stats_config.py index 2b93335f85a7..459478d29382 100644 --- a/products/experiments/backend/hogql_queries/test/test_stats_config.py +++ b/products/experiments/backend/hogql_queries/test/test_stats_config.py @@ -18,6 +18,7 @@ from products.experiments.backend.hogql_queries.utils import ( get_bayesian_experiment_result, get_frequentist_experiment_result, + resolve_baseline_variant_key, split_baseline_and_test_variants, ) from products.experiments.backend.models.experiment import Experiment @@ -400,6 +401,41 @@ def test_experiment_query_runner_reads_baseline_from_stats_config(self, _name, s self.assertEqual(runner.baseline_variant_key, expected_baseline) + @parameterized.expand( + [ + # configured baseline present -> used as-is + ("configured_present", ["control", "test"], "test", "test"), + ("configured_present_control", ["control", "test"], "control", "control"), + # configured baseline missing, control present -> falls back to control + ("missing_falls_back_to_control", ["control", "test-a", "test-b"], "removed", "control"), + # configured baseline missing and no control -> falls back to first variant + ("missing_no_control_first_variant", ["variant-a", "variant-b"], "removed", "variant-a"), + # no variants at all -> returns the configured key unchanged (degenerate) + ("no_variants", [], "control", "control"), + # default (None configured) resolves to control when present + ("none_configured_control_present", ["control", "test"], None, "control"), + ] + ) + def test_resolve_baseline_variant_key(self, _name, variant_keys, configured, expected): + if configured is None: + self.assertEqual(resolve_baseline_variant_key(variant_keys), expected) + else: + self.assertEqual(resolve_baseline_variant_key(variant_keys, configured), expected) + + def test_resolve_then_split_recovers_when_baseline_removed(self): + # When the configured baseline variant was removed from the flag, resolving first + # keeps split_baseline_and_test_variants working instead of raising "No control variant". + variants = [ + self.create_variant("control", sum_val=100.0, sum_squares=10500.0, samples=1000), + self.create_variant("test", sum_val=120.0, sum_squares=14500.0, samples=1000), + ] + resolved = resolve_baseline_variant_key([v.key for v in variants], "removed-baseline") + self.assertEqual(resolved, "control") + + baseline, test_variants = split_baseline_and_test_variants(variants, resolved) + self.assertEqual(baseline.key, "control") + self.assertEqual([v.key for v in test_variants], ["test"]) + class TestSequentialStatsConfig(APIBaseTest): """Verify sequential_testing flags propagate from stats_config (and team defaults) into the engine.""" diff --git a/products/experiments/backend/hogql_queries/utils.py b/products/experiments/backend/hogql_queries/utils.py index 78c16bae514f..b02b3891ce54 100644 --- a/products/experiments/backend/hogql_queries/utils.py +++ b/products/experiments/backend/hogql_queries/utils.py @@ -107,6 +107,26 @@ def get_experiment_stats_method(experiment) -> str: return stats_method +def resolve_baseline_variant_key(variant_keys: list[str], configured_key: str = CONTROL_VARIANT_KEY) -> str: + """Resolve the effective baseline variant key against the variants actually present. + + The configured baseline (``stats_config.baseline_variant_key``) can drift out of sync + with the flag's variants — e.g. the chosen baseline variant is renamed or removed on the + feature flag directly, bypassing the experiment-update validation that normally blocks + dangling baselines. Rather than erroring the whole analysis when that happens, fall back + in order of preference: the configured key, then the conventional ``control`` variant, + then the first available variant. The experiment-update path still validates and rejects + dangling baselines; this is the read-time safety net. + """ + if configured_key in variant_keys: + return configured_key + if CONTROL_VARIANT_KEY in variant_keys: + return CONTROL_VARIANT_KEY + if variant_keys: + return variant_keys[0] + return configured_key + + def split_baseline_and_test_variants( variants: list[V], baseline_key: str = CONTROL_VARIANT_KEY, diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py index 3c13585f4d65..757bc1813acd 100644 --- a/products/experiments/backend/presentation/serializers.py +++ b/products/experiments/backend/presentation/serializers.py @@ -360,6 +360,7 @@ def to_representation(self, instance): instance.exposure_criteria, only_count_matured_users=instance.only_count_matured_users, excluded_variants=(instance.parameters or {}).get("excluded_variants"), + baseline_variant_key=(instance.stats_config or {}).get("baseline_variant_key"), ) return data diff --git a/products/experiments/backend/recalculation.py b/products/experiments/backend/recalculation.py index 0e9e7246e7f6..67d1b74dfdc1 100644 --- a/products/experiments/backend/recalculation.py +++ b/products/experiments/backend/recalculation.py @@ -232,6 +232,8 @@ def _recalc_fingerprints_for_run(experiment: Experiment, recalc: ExperimentMetri stats_method, experiment.exposure_criteria, only_count_matured_users=experiment.only_count_matured_users, + excluded_variants=(experiment.parameters or {}).get("excluded_variants"), + baseline_variant_key=(experiment.stats_config or {}).get("baseline_variant_key"), ) fingerprints[metric_uuid] = compute_recalc_fingerprint(config_fp, str(recalc.id)) return fingerprints diff --git a/products/experiments/backend/temporal/recalculation_logic.py b/products/experiments/backend/temporal/recalculation_logic.py index 5f66073bdada..9165e74cd34b 100644 --- a/products/experiments/backend/temporal/recalculation_logic.py +++ b/products/experiments/backend/temporal/recalculation_logic.py @@ -442,6 +442,8 @@ def _calculate_experiment_metric_for_recalculation_sync( get_experiment_stats_method(experiment), experiment.exposure_criteria, only_count_matured_users=experiment.only_count_matured_users, + excluded_variants=(experiment.parameters or {}).get("excluded_variants"), + baseline_variant_key=(experiment.stats_config or {}).get("baseline_variant_key"), ) recalc_fp = compute_recalc_fingerprint(config_fp, recalculation_id) diff --git a/products/experiments/backend/test/test_experiment_service.py b/products/experiments/backend/test/test_experiment_service.py index 45007ec49ac5..26caefd6e592 100644 --- a/products/experiments/backend/test/test_experiment_service.py +++ b/products/experiments/backend/test/test_experiment_service.py @@ -5030,6 +5030,54 @@ def test_update_experiment_revalidates_baseline_when_variants_change(self) -> No }, ) + @parameterized.expand( + [ + ("draft",), + ("running",), + ("completed",), + ] + ) + def test_update_baseline_variant_key_allowed_across_states(self, state: str) -> None: + key = f"baseline-switch-{state}" + name = f"Switch baseline {state}" + if state == "completed": + experiment = self._create_ended_experiment(name=name, feature_flag_key=key) + else: + experiment = self._create_launchable_experiment(name=name, feature_flag_key=key) + if state == "running": + self._service().launch_experiment(experiment) + experiment.refresh_from_db() + + # The default flag has control/test variants; switching the analysis baseline to an + # existing variant is allowed regardless of lifecycle state. + self._service().update_experiment( + experiment, + {"stats_config": {**(experiment.stats_config or {}), "baseline_variant_key": "test"}}, + ) + experiment.refresh_from_db() + assert experiment.stats_config is not None + assert experiment.stats_config["baseline_variant_key"] == "test" + + def test_duplicate_experiment_preserves_baseline_variant_key(self) -> None: + self._create_flag( + key="baseline-dup-source", + variants=[ + {"key": "control", "name": "Control", "rollout_percentage": 34}, + {"key": "variant-a", "name": "Variant A", "rollout_percentage": 33}, + {"key": "variant-b", "name": "Variant B", "rollout_percentage": 33}, + ], + ) + service = self._service() + source = service.create_experiment( + name="Baseline dup source", + feature_flag_key="baseline-dup-source", + stats_config={"baseline_variant_key": "variant-a"}, + ) + + clone = service.duplicate_experiment(source, feature_flag_key="baseline-dup-clone") + assert clone.stats_config is not None + assert clone.stats_config["baseline_variant_key"] == "variant-a" + class TestValidateExperimentParametersExcludedVariants: def _base_params(self) -> dict[str, Any]: diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.py index fe9c9f804339..627856eb4021 100644 --- a/products/experiments/backend/test/test_presentation_api.py +++ b/products/experiments/backend/test/test_presentation_api.py @@ -5440,6 +5440,49 @@ def test_web_experiment_activity_logging_excludes_parameters_through_main_endpoi self.assertIn("description", change_fields) self.assertNotIn("parameters", change_fields) + def test_experiment_baseline_variant_change_is_logged(self): + feature_flag = FeatureFlag.objects.create( + team=self.team, + created_by=self.user, + name="Baseline activity flag", + key="baseline-activity-flag", + filters={ + "groups": [{"properties": [], "rollout_percentage": 100}], + "multivariate": { + "variants": [ + {"key": "control", "name": "Control", "rollout_percentage": 50}, + {"key": "test", "name": "Test", "rollout_percentage": 50}, + ] + }, + }, + ) + experiment = Experiment.objects.create( + team=self.team, + created_by=self.user, + name="Baseline activity experiment", + feature_flag=feature_flag, + stats_config={"method": "bayesian"}, + ) + + update_response = self.client.patch( + f"/api/projects/{self.team.id}/experiments/{experiment.id}/", + {"stats_config": {"method": "bayesian", "baseline_variant_key": "test"}}, + format="json", + ) + self.assertEqual(update_response.status_code, status.HTTP_200_OK) + + experiment.refresh_from_db() + assert experiment.stats_config is not None + self.assertEqual(experiment.stats_config["baseline_variant_key"], "test") + + # Switching the baseline is an auditable change: it surfaces as a stats_config diff. + activity_log = ActivityLog.objects.filter( + scope="Experiment", item_id=str(experiment.id), activity="updated" + ).latest("created_at") + assert activity_log.detail is not None + change_fields = [change["field"] for change in activity_log.detail["changes"]] + self.assertIn("stats_config", change_fields) + def test_experiment_saved_metric_activity_logging_shows_correct_user_for_updates(self): """Test that experiment saved metric activity logs show the correct user for both creation and updates."""