diff --git a/frontend/src/scenes/experiments/ExperimentView/Info.tsx b/frontend/src/scenes/experiments/ExperimentView/Info.tsx index 3697bbc5cf7a..8b0309e4ba8e 100644 --- a/frontend/src/scenes/experiments/ExperimentView/Info.tsx +++ b/frontend/src/scenes/experiments/ExperimentView/Info.tsx @@ -18,6 +18,7 @@ import { CONCLUSION_DISPLAY_CONFIG } from '../constants' import { experimentLogic, previousRefreshAnalytics } from '../experimentLogic' import { getExperimentStatus, isExperimentPaused } from '../experimentsLogic' import { modalsLogic } from '../modalsLogic' +import { formatStatsLevelPercent, getExperimentStatsLevel } from '../utils' import { ExperimentDuration } from './ExperimentDuration' import { ExperimentReloadAction } from './ExperimentReloadAction' import { RunningTime } from './RunningTime' @@ -134,9 +135,7 @@ export function Info(): JSX.Element { {statsMethod === ExperimentStatsMethod.Bayesian ? 'Bayesian' : 'Frequentist'} {' / '} - {statsMethod === ExperimentStatsMethod.Bayesian - ? `${((experiment.stats_config?.bayesian?.ci_level ?? 0.95) * 100).toFixed(0)}%` - : `${((1 - (experiment.stats_config?.frequentist?.alpha ?? 0.05)) * 100).toFixed(0)}%`} + {formatStatsLevelPercent(getExperimentStatsLevel(experiment))} diff --git a/frontend/src/scenes/experiments/ExperimentView/SettingsTab.tsx b/frontend/src/scenes/experiments/ExperimentView/SettingsTab.tsx index 314c1e091c8f..cf7094309031 100644 --- a/frontend/src/scenes/experiments/ExperimentView/SettingsTab.tsx +++ b/frontend/src/scenes/experiments/ExperimentView/SettingsTab.tsx @@ -12,6 +12,7 @@ import { ExperimentStatsMethod, PropertyFilterType, PropertyOperator } from '~/t import { DEFAULT_LOOKBACK_DAYS } from '../constants' import { experimentLogic } from '../experimentLogic' import { modalsLogic } from '../modalsLogic' +import { formatStatsLevelPercent, getExperimentStatsLevel } from '../utils' import { getCupedSelection, resolveCupedEnabled, resolveCupedLookbackDays } from './cuped' import { CupedModal } from './CupedModal' import { resolveSequentialEnabled } from './sequential' @@ -25,9 +26,7 @@ export function SettingsTab(): JSX.Element { const isBayesian = statsMethod === ExperimentStatsMethod.Bayesian - const confidenceDisplay = isBayesian - ? `${((experiment.stats_config?.bayesian?.ci_level ?? 0.95) * 100).toFixed(0)}%` - : `${((1 - (experiment.stats_config?.frequentist?.alpha ?? 0.05)) * 100).toFixed(0)}%` + const confidenceDisplay = formatStatsLevelPercent(getExperimentStatsLevel(experiment)) const teamDefaultCupedEnabled = experimentsConfig?.default_cuped_enabled ?? false const teamDefaultCupedLookbackDays = experimentsConfig?.default_cuped_lookback_days ?? null diff --git a/frontend/src/scenes/experiments/ExperimentView/StatsMethodModal.tsx b/frontend/src/scenes/experiments/ExperimentView/StatsMethodModal.tsx index 809615e46c7a..6eb382c12d71 100644 --- a/frontend/src/scenes/experiments/ExperimentView/StatsMethodModal.tsx +++ b/frontend/src/scenes/experiments/ExperimentView/StatsMethodModal.tsx @@ -11,6 +11,7 @@ import { StatsMethodSelector } from '../components/StatsMethodSelector' import { CONFIDENCE_LEVEL_OPTIONS } from '../constants' import { experimentLogic } from '../experimentLogic' import { modalsLogic } from '../modalsLogic' +import { getExperimentStatsLevel } from '../utils' import { DEFAULT_SEQUENTIAL_TUNING_PARAMETER, MAX_SEQUENTIAL_TUNING_PARAMETER, @@ -35,9 +36,7 @@ export function StatsMethodModal(): JSX.Element { // For Bayesian: ci_level (default 0.95) // For Frequentist: confidence = 1 - alpha (default alpha 0.05 = 95% confidence) - const currentConfidenceLevel = isBayesian - ? (experiment.stats_config?.bayesian?.ci_level ?? 0.95) - : 1 - (experiment.stats_config?.frequentist?.alpha ?? 0.05) + const currentConfidenceLevel = getExperimentStatsLevel(experiment) const handleConfidenceLevelChange = (value: number): void => { if (isBayesian) { diff --git a/frontend/src/scenes/experiments/MetricsView/new/HowToReadTooltip.tsx b/frontend/src/scenes/experiments/MetricsView/new/HowToReadTooltip.tsx index 638c95488c22..8ae64206bce2 100644 --- a/frontend/src/scenes/experiments/MetricsView/new/HowToReadTooltip.tsx +++ b/frontend/src/scenes/experiments/MetricsView/new/HowToReadTooltip.tsx @@ -10,11 +10,14 @@ import { themeLogic } from '~/layout/navigation-3000/themeLogic' import { ExperimentStatsMethod } from '~/types' import { experimentLogic } from '../../experimentLogic' +import { formatStatsLevelPercent, getExperimentStatsLevel } from '../../utils' export function HowToReadTooltip(): JSX.Element { - const { statsMethod } = useValues(experimentLogic) + const { experiment, statsMethod } = useValues(experimentLogic) const { isDarkModeOn } = useValues(themeLogic) + const statsLevel = formatStatsLevelPercent(getExperimentStatsLevel(experiment)) + return ( <> @@ -64,8 +67,8 @@ export function HowToReadTooltip(): JSX.Element {

The bars show{' '} {statsMethod === ExperimentStatsMethod.Bayesian - ? '95% credible intervals' - : '95% confidence intervals'} + ? `${statsLevel} credible intervals` + : `${statsLevel} confidence intervals`} . When an interval doesn't cross the 0% line, the result is significant.

= [ { key: 'variant', @@ -233,9 +248,7 @@ export function ResultDetails({ }, { key: 'interval', - title: result.variant_results?.[0] - ? `${getIntervalLabel(result.variant_results[0])} (95%)` - : 'Confidence interval (95%)', + title: intervalColumnTitle, render: (_, item: ExperimentVariantResult & { key: string }) => { if (item.key === baselineKey) { return '—' diff --git a/frontend/src/scenes/experiments/constants.ts b/frontend/src/scenes/experiments/constants.ts index e3ad9a2fcd30..4898051476e2 100644 --- a/frontend/src/scenes/experiments/constants.ts +++ b/frontend/src/scenes/experiments/constants.ts @@ -36,6 +36,12 @@ export const CONFIDENCE_LEVEL_OPTIONS = [ { value: 0.99, label: '99%' }, ] +// Defaults used when an experiment hasn't explicitly configured its statistics level. +// Bayesian stores the credible-interval level directly (ci_level); frequentist stores the +// significance level (alpha), so the displayed confidence level is 1 - alpha. +export const DEFAULT_BAYESIAN_CI_LEVEL = 0.95 +export const DEFAULT_FREQUENTIST_ALPHA = 0.05 + export const EXPERIMENT_MIN_EXPOSURES_FOR_RESULTS = 50 export const EXPERIMENT_MIN_METRIC_VALUE_FOR_RESULTS = 10 diff --git a/frontend/src/scenes/experiments/utils.test.ts b/frontend/src/scenes/experiments/utils.test.ts index 44cd97a86170..2e463d2fdc5f 100644 --- a/frontend/src/scenes/experiments/utils.test.ts +++ b/frontend/src/scenes/experiments/utils.test.ts @@ -19,6 +19,7 @@ import { AccessControlLevel, Experiment, ExperimentMetricMathType, + ExperimentStatsMethod, FeatureFlagBucketingIdentifier, FeatureFlagEvaluationRuntime, FeatureFlagType, @@ -32,7 +33,9 @@ import { exposureConfigToFilter, featureFlagEligibleForExperiment, filterToExposureConfig, + formatStatsLevelPercent, getEventCountQuery, + getExperimentStatsLevel, getOrderedMetricsWithResults, getViewRecordingFilters, getViewRecordingFiltersLegacy, @@ -1382,4 +1385,93 @@ describe('getEventCountQuery', () => { expect(query).toBeNull() }) + + describe('getExperimentStatsLevel', () => { + const buildExperiment = (statsConfig: Experiment['stats_config']): Experiment => + ({ stats_config: statsConfig }) as Experiment + + it.each([ + ['bayesian 90%', { method: ExperimentStatsMethod.Bayesian, bayesian: { ci_level: 0.9 } }, 0.9], + ['bayesian 95%', { method: ExperimentStatsMethod.Bayesian, bayesian: { ci_level: 0.95 } }, 0.95], + ['bayesian 99%', { method: ExperimentStatsMethod.Bayesian, bayesian: { ci_level: 0.99 } }, 0.99], + [ + 'bayesian missing ci_level defaults to 95%', + { method: ExperimentStatsMethod.Bayesian, bayesian: {} }, + 0.95, + ], + ['bayesian missing bayesian key defaults to 95%', { method: ExperimentStatsMethod.Bayesian }, 0.95], + [ + 'frequentist 90% (alpha 0.1)', + { method: ExperimentStatsMethod.Frequentist, frequentist: { alpha: 0.1 } }, + 0.9, + ], + [ + 'frequentist 95% (alpha 0.05)', + { method: ExperimentStatsMethod.Frequentist, frequentist: { alpha: 0.05 } }, + 0.95, + ], + [ + 'frequentist 99% (alpha 0.01)', + { method: ExperimentStatsMethod.Frequentist, frequentist: { alpha: 0.01 } }, + 0.99, + ], + [ + 'frequentist missing alpha defaults to 95%', + { method: ExperimentStatsMethod.Frequentist, frequentist: {} }, + 0.95, + ], + [ + 'frequentist missing frequentist key defaults to 95%', + { method: ExperimentStatsMethod.Frequentist }, + 0.95, + ], + ])('resolves the configured level for %s', (_name, statsConfig, expected) => { + expect(getExperimentStatsLevel(buildExperiment(statsConfig as Experiment['stats_config']))).toBeCloseTo( + expected, + 10 + ) + }) + + it.each([ + ['undefined stats_config', undefined], + ['null stats_config', null], + ['empty stats_config', {}], + ])('defaults to Bayesian 95% when %s', (_name, statsConfig) => { + expect(getExperimentStatsLevel(buildExperiment(statsConfig as Experiment['stats_config']))).toBeCloseTo( + 0.95, + 10 + ) + }) + + it('honors an explicit statsMethod override (e.g. matching a result method)', () => { + // Experiment is configured Bayesian, but the override forces frequentist resolution. + const experiment = buildExperiment({ + method: ExperimentStatsMethod.Bayesian, + bayesian: { ci_level: 0.9 }, + frequentist: { alpha: 0.2 }, + } as Experiment['stats_config']) + + expect(getExperimentStatsLevel(experiment)).toBeCloseTo(0.9, 10) + expect(getExperimentStatsLevel(experiment, ExperimentStatsMethod.Frequentist)).toBeCloseTo(0.8, 10) + }) + }) + + describe('formatStatsLevelPercent', () => { + it.each([ + [0.9, '90%'], + [0.95, '95%'], + [0.99, '99%'], + [0.8, '80%'], + [0.5, '50%'], + ])('formats %p as %s', (level, expected) => { + expect(formatStatsLevelPercent(level)).toBe(expected) + }) + + it('formats the label shown for a Bayesian 90% experiment', () => { + const experiment = { + stats_config: { method: ExperimentStatsMethod.Bayesian, bayesian: { ci_level: 0.9 } }, + } as Experiment + expect(formatStatsLevelPercent(getExperimentStatsLevel(experiment))).toBe('90%') + }) + }) }) diff --git a/frontend/src/scenes/experiments/utils.ts b/frontend/src/scenes/experiments/utils.ts index a85cf7aa97d5..3aa762f4c2af 100644 --- a/frontend/src/scenes/experiments/utils.ts +++ b/frontend/src/scenes/experiments/utils.ts @@ -34,6 +34,7 @@ import { Experiment, ExperimentMetricGoal, ExperimentMetricMathType, + ExperimentStatsMethod, FeatureFlagType, FilterType, FunnelConversionWindowTimeUnit, @@ -45,11 +46,39 @@ import { UniversalFiltersGroupValue, } from '~/types' -import { EXPERIMENT_VARIANT_MULTIPLE } from './constants' +import { DEFAULT_BAYESIAN_CI_LEVEL, DEFAULT_FREQUENTIST_ALPHA, EXPERIMENT_VARIANT_MULTIPLE } from './constants' import { SharedMetric } from './SharedMetrics/sharedMetricLogic' const MULTIPLE_VARIANT_WARNING_THRESHOLD = 0.5 // on the 0-100 scale (0.5 = 0.5%) +/** + * Resolve an experiment's configured statistics method, defaulting to Bayesian when unset. + */ +export function getExperimentStatsMethod(experiment: Experiment): ExperimentStatsMethod { + return experiment.stats_config?.method || ExperimentStatsMethod.Bayesian +} + +/** + * Resolve the configured statistics level (as a 0-1 fraction) for an experiment. Bayesian stores + * the credible-interval level directly (`ci_level`); frequentist stores the significance level + * (`alpha`), so the displayed confidence level is `1 - alpha`. Falls back to the 95% default when + * the setting is unset. Pass `statsMethod` to resolve the level for a specific method (e.g. to match + * a result's method) instead of the experiment's configured method. + */ +export function getExperimentStatsLevel(experiment: Experiment, statsMethod?: ExperimentStatsMethod): number { + const method = statsMethod ?? getExperimentStatsMethod(experiment) + return method === ExperimentStatsMethod.Bayesian + ? (experiment.stats_config?.bayesian?.ci_level ?? DEFAULT_BAYESIAN_CI_LEVEL) + : 1 - (experiment.stats_config?.frequentist?.alpha ?? DEFAULT_FREQUENTIST_ALPHA) +} + +/** + * Format a 0-1 statistics level as a whole-number percentage, e.g. 0.9 -> "90%". + */ +export function formatStatsLevelPercent(level: number): string { + return `${(level * 100).toFixed(0)}%` +} + export function filterLowMultipleVariant(variants: T[]): T[] { return variants.filter( (v) => v.variant !== EXPERIMENT_VARIANT_MULTIPLE || v.percentage > MULTIPLE_VARIANT_WARNING_THRESHOLD diff --git a/products/experiments/backend/hogql_queries/test/test_stats_config.py b/products/experiments/backend/hogql_queries/test/test_stats_config.py index 2b93335f85a7..c75adb85d706 100644 --- a/products/experiments/backend/hogql_queries/test/test_stats_config.py +++ b/products/experiments/backend/hogql_queries/test/test_stats_config.py @@ -228,6 +228,104 @@ def test_numeric_validation_ci_level_out_of_range_uses_default(self) -> None: variant = cast(ExperimentVariantResultBayesian, result.variant_results[0]) assert variant.credible_interval is not None + @parameterized.expand( + [ + ("config_none", None), + ("config_empty", {}), + ("bayesian_key_empty", {"bayesian": {}}), + ] + ) + def test_bayesian_missing_ci_level_matches_explicit_95(self, _name, stats_config) -> None: + # The frontend labels the credible interval "95%" when the level is unset, so the calculated + # interval for a missing config must match the explicit 0.95 configuration exactly. + metric = self.create_mean_metric() + control = self.create_variant("control", sum_val=1000.0, sum_squares=105000.0, samples=1000) + test = self.create_variant("test", sum_val=1200.0, sum_squares=145000.0, samples=1000) + + result_default = get_bayesian_experiment_result( + metric=metric, control_variant=control, test_variants=[test], stats_config=stats_config + ) + result_95 = get_bayesian_experiment_result( + metric=metric, control_variant=control, test_variants=[test], stats_config={"bayesian": {"ci_level": 0.95}} + ) + + assert result_default.variant_results is not None and result_95.variant_results is not None + default_ci = cast(ExperimentVariantResultBayesian, result_default.variant_results[0]).credible_interval + explicit_ci = cast(ExperimentVariantResultBayesian, result_95.variant_results[0]).credible_interval + assert default_ci is not None and explicit_ci is not None + self.assertAlmostEqual(default_ci[0], explicit_ci[0], places=10) + self.assertAlmostEqual(default_ci[1], explicit_ci[1], places=10) + + @parameterized.expand([("level_90", 0.90), ("level_95", 0.95), ("level_99", 0.99)]) + def test_bayesian_supported_levels_widen_with_level(self, _name, ci_level) -> None: + metric = self.create_mean_metric() + control = self.create_variant("control", sum_val=1000.0, sum_squares=105000.0, samples=1000) + test = self.create_variant("test", sum_val=1200.0, sum_squares=145000.0, samples=1000) + + result = get_bayesian_experiment_result( + metric=metric, + control_variant=control, + test_variants=[test], + stats_config={"bayesian": {"ci_level": ci_level}}, + ) + result_90 = get_bayesian_experiment_result( + metric=metric, control_variant=control, test_variants=[test], stats_config={"bayesian": {"ci_level": 0.90}} + ) + + assert result.variant_results is not None and result_90.variant_results is not None + ci = cast(ExperimentVariantResultBayesian, result.variant_results[0]).credible_interval + ci_90 = cast(ExperimentVariantResultBayesian, result_90.variant_results[0]).credible_interval + assert ci is not None and ci_90 is not None + # A higher credible level can never produce a narrower interval than the 90% baseline. + self.assertGreaterEqual((ci[1] - ci[0]) + 1e-9, ci_90[1] - ci_90[0]) + + @parameterized.expand( + [ + ("config_none", None), + ("config_empty", {}), + ("frequentist_key_empty", {"frequentist": {}}), + ] + ) + def test_frequentist_missing_alpha_matches_explicit_005(self, _name, stats_config) -> None: + # Unset alpha is labeled "95%" on the frontend, so it must match the explicit alpha=0.05 interval. + metric = self.create_mean_metric() + control = self.create_variant("control", sum_val=1000.0, sum_squares=105000.0, samples=1000) + test = self.create_variant("test", sum_val=1200.0, sum_squares=145000.0, samples=1000) + + result_default = get_frequentist_experiment_result( + metric=metric, control_variant=control, test_variants=[test], stats_config=stats_config + ) + result_005 = get_frequentist_experiment_result( + metric=metric, control_variant=control, test_variants=[test], stats_config={"frequentist": {"alpha": 0.05}} + ) + + assert result_default.variant_results is not None and result_005.variant_results is not None + default_ci = cast(ExperimentVariantResultFrequentist, result_default.variant_results[0]).confidence_interval + explicit_ci = cast(ExperimentVariantResultFrequentist, result_005.variant_results[0]).confidence_interval + assert default_ci is not None and explicit_ci is not None + self.assertAlmostEqual(default_ci[0], explicit_ci[0], places=10) + self.assertAlmostEqual(default_ci[1], explicit_ci[1], places=10) + + @parameterized.expand([("alpha_10", 0.10), ("alpha_05", 0.05), ("alpha_01", 0.01)]) + def test_frequentist_supported_levels_widen_as_alpha_shrinks(self, _name, alpha) -> None: + metric = self.create_mean_metric() + control = self.create_variant("control", sum_val=1000.0, sum_squares=105000.0, samples=1000) + test = self.create_variant("test", sum_val=1200.0, sum_squares=145000.0, samples=1000) + + result = get_frequentist_experiment_result( + metric=metric, control_variant=control, test_variants=[test], stats_config={"frequentist": {"alpha": alpha}} + ) + result_10 = get_frequentist_experiment_result( + metric=metric, control_variant=control, test_variants=[test], stats_config={"frequentist": {"alpha": 0.10}} + ) + + assert result.variant_results is not None and result_10.variant_results is not None + ci = cast(ExperimentVariantResultFrequentist, result.variant_results[0]).confidence_interval + ci_10 = cast(ExperimentVariantResultFrequentist, result_10.variant_results[0]).confidence_interval + assert ci is not None and ci_10 is not None + # Smaller alpha (higher confidence) can never produce a narrower interval than the 90% baseline. + self.assertGreaterEqual((ci[1] - ci[0]) + 1e-9, ci_10[1] - ci_10[0]) + @parameterized.expand(INSUFFICIENT_DATA_CASES) def test_frequentist_insufficient_data_returns_raw_values_without_stats(self, _name, data): metric = self.create_mean_metric() diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.py index fe9c9f804339..307cb0b05b45 100644 --- a/products/experiments/backend/test/test_presentation_api.py +++ b/products/experiments/backend/test/test_presentation_api.py @@ -354,6 +354,42 @@ def test_creating_updating_basic_experiment(self): assert experiment.end_date is not None self.assertEqual(experiment.end_date.strftime("%Y-%m-%dT%H:%M"), end_date) + @parameterized.expand( + [ + ("bayesian_90", {"method": "bayesian", "bayesian": {"ci_level": 0.9}}), + ("bayesian_95", {"method": "bayesian", "bayesian": {"ci_level": 0.95}}), + ("bayesian_99", {"method": "bayesian", "bayesian": {"ci_level": 0.99}}), + ("bayesian_default_unset", {"method": "bayesian"}), + ("frequentist_90", {"method": "frequentist", "frequentist": {"alpha": 0.1}}), + ("frequentist_99", {"method": "frequentist", "frequentist": {"alpha": 0.01}}), + ] + ) + def test_stats_config_level_round_trips_through_api(self, name: str, stats_config: dict) -> None: + # The displayed credible/confidence level is derived on the frontend from stats_config, + # so the configured level must survive the create + read serialization round-trip. + create_response = self.client.post( + f"/api/projects/{self.team.id}/experiments/", + { + "name": f"Experiment {name}", + "feature_flag_key": f"stats-config-{name}", + "parameters": None, + "filters": {"events": [{"order": 0, "id": "$pageview"}]}, + "stats_config": stats_config, + }, + ) + self.assertEqual(create_response.status_code, status.HTTP_201_CREATED) + experiment_id = create_response.json()["id"] + + get_response = self.client.get(f"/api/projects/{self.team.id}/experiments/{experiment_id}/") + self.assertEqual(get_response.status_code, status.HTTP_200_OK) + returned_config = get_response.json()["stats_config"] + + self.assertEqual(returned_config.get("method"), stats_config["method"]) + if "bayesian" in stats_config: + self.assertEqual(returned_config["bayesian"]["ci_level"], stats_config["bayesian"]["ci_level"]) + if "frequentist" in stats_config: + self.assertEqual(returned_config["frequentist"]["alpha"], stats_config["frequentist"]["alpha"]) + @patch("products.experiments.backend.experiment_service.report_user_action") def test_creating_experiment_reports_user_action(self, mock_report_user_action): ff_key = "tracked-experiment"