[{"data":1,"prerenderedAt":1389},["ShallowReactive",2],{"navigation_docs":3,"-getting-started-experiments-how-results-are-computed":339,"-getting-started-experiments-how-results-are-computed-surround":1384},[4,10,71,97,249,320],{"title":5,"path":6,"stem":7,"icon":8,"exact":9},"Introduction","\u002F","0.index","i-lucide-book-open",true,{"title":11,"icon":12,"modifiedAt":13,"path":14,"stem":15,"children":16,"page":12},"Getting Started",false,"2026-04-30T19:15:33.000Z","\u002Fgetting-started","1.getting-started",[17,22,27,32,37,42,47,61,66],{"title":18,"path":19,"stem":20,"icon":21},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F1.quickstart","i-lucide-fast-forward",{"title":23,"path":24,"stem":25,"icon":26},"Core Concepts","\u002Fgetting-started\u002Fcore-concepts","1.getting-started\u002F2.core-concepts","i-lucide-workflow",{"title":28,"path":29,"stem":30,"icon":31},"Configs Anatomy","\u002Fgetting-started\u002Fconfigs","1.getting-started\u002F3.configs","i-lucide-file-cog",{"title":33,"path":34,"stem":35,"icon":36},"Architecture","\u002Fgetting-started\u002Farchitecture","1.getting-started\u002F4.architecture","i-lucide-layers",{"title":38,"path":39,"stem":40,"icon":41},"JSON Schema Validation","\u002Fgetting-started\u002Fjson-schema-validation","1.getting-started\u002F5.json-schema-validation","i-lucide-braces",{"title":43,"path":44,"stem":45,"icon":46},"Testing Rules","\u002Fgetting-started\u002Ftesting-targeting-rules","1.getting-started\u002F6.testing-targeting-rules","i-lucide-flask-conical",{"title":48,"modifiedAt":49,"icon":50,"defaultOpen":12,"path":51,"stem":52,"children":53},"Experiments","2026-11-10T01:16:18.000Z","i-lucide-flask-round","\u002Fgetting-started\u002Fexperiments","1.getting-started\u002F7.experiments\u002Findex",[54,56],{"title":55,"path":51,"stem":52,"icon":8},"Overview",{"title":57,"path":58,"stem":59,"icon":60},"How results are computed","\u002Fgetting-started\u002Fexperiments\u002Fhow-results-are-computed","1.getting-started\u002F7.experiments\u002F1.how-results-are-computed","i-lucide-sigma",{"title":62,"path":63,"stem":64,"icon":65},"Segments","\u002Fgetting-started\u002Fsegments","1.getting-started\u002F8.segments","i-lucide-squares-intersect",{"title":67,"path":68,"stem":69,"icon":70},"Pricing & Billing","\u002Fgetting-started\u002Fpricing-and-billing","1.getting-started\u002F9.pricing-and-billing","i-lucide-credit-card",{"title":72,"modifiedAt":73,"defaultOpen":9,"path":74,"stem":75,"children":76,"page":12},"Monitoring","2026-09-24T23:33:20.000Z","\u002Fmonitoring","2.monitoring",[77,82,87,92],{"title":78,"path":79,"stem":80,"icon":81},"Activity","\u002Fmonitoring\u002Factivity","2.monitoring\u002F1.activity","i-lucide-activity",{"title":83,"path":84,"stem":85,"icon":86},"Lifecycle","\u002Fmonitoring\u002Flifecycle","2.monitoring\u002F2.lifecycle","i-lucide-recycle",{"title":88,"path":89,"stem":90,"icon":91},"Alerts","\u002Fmonitoring\u002Falerts","2.monitoring\u002F3.alerts","i-lucide-bell",{"title":93,"path":94,"stem":95,"icon":96},"Dynamic Detection","\u002Fmonitoring\u002Fdynamic-detection","2.monitoring\u002F4.dynamic-detection","i-lucide-hat-glasses",{"title":98,"modifiedAt":73,"defaultOpen":9,"path":99,"stem":100,"children":101,"page":12},"SDKs","\u002Fsdks","3.sdks",[102,107,128,153,169,200,244],{"title":103,"path":104,"stem":105,"icon":106},"Available SDKs","\u002Fsdks\u002Favailable-sdks","3.sdks\u002F1.available-sdks","i-lucide-list",{"title":108,"modifiedAt":73,"icon":109,"defaultOpen":12,"path":110,"stem":111,"children":112,"page":12},"Browser SDKs","i-lucide-monitor","\u002Fsdks\u002Fbrowser","3.sdks\u002F2.browser",[113,118,123],{"title":114,"path":115,"stem":116,"icon":117},"Javascript","\u002Fsdks\u002Fbrowser\u002Fjavascript","3.sdks\u002F2.browser\u002F1.javascript","cd-logos-javascript",{"title":119,"path":120,"stem":121,"icon":122},"React","\u002Fsdks\u002Fbrowser\u002Freact","3.sdks\u002F2.browser\u002F2.react","cd-logos-react-light",{"title":124,"path":125,"stem":126,"icon":127},"Vue","\u002Fsdks\u002Fbrowser\u002Fvue","3.sdks\u002F2.browser\u002F3.vue","cd-logos-vue",{"title":129,"modifiedAt":73,"icon":130,"defaultOpen":12,"path":131,"stem":132,"children":133,"page":12},"Mobile SDKs","i-lucide-smartphone","\u002Fsdks\u002Fmobile","3.sdks\u002F3.mobile",[134,138,143,148],{"title":135,"path":136,"stem":137,"icon":122},"React Native","\u002Fsdks\u002Fmobile\u002Freact-native","3.sdks\u002F3.mobile\u002F1.react-native",{"title":139,"path":140,"stem":141,"icon":142},"Flutter","\u002Fsdks\u002Fmobile\u002Fflutter","3.sdks\u002F3.mobile\u002F2.flutter","cd-logos-flutter",{"title":144,"path":145,"stem":146,"icon":147},"Swift","\u002Fsdks\u002Fmobile\u002Fswift","3.sdks\u002F3.mobile\u002F3.swift","cd-logos-swift",{"title":149,"path":150,"stem":151,"icon":152},"Android","\u002Fsdks\u002Fmobile\u002Fandroid","3.sdks\u002F3.mobile\u002F4.android","cd-logos-android",{"title":154,"modifiedAt":73,"icon":155,"defaultOpen":12,"path":156,"stem":157,"children":158,"page":12},"Meta-framework SDKs","i-lucide-monitor-smartphone","\u002Fsdks\u002Fmeta","3.sdks\u002F4.meta",[159,164],{"title":160,"path":161,"stem":162,"icon":163},"Next.js","\u002Fsdks\u002Fmeta\u002Fnextjs","3.sdks\u002F4.meta\u002F1.nextjs","cd-logos-nextjs",{"title":165,"path":166,"stem":167,"icon":168},"Nuxt","\u002Fsdks\u002Fmeta\u002Fnuxt","3.sdks\u002F4.meta\u002F2.nuxt","cd-logos-nuxt",{"title":170,"modifiedAt":73,"icon":171,"defaultOpen":12,"path":172,"stem":173,"children":174,"page":12},"Server SDKs","i-lucide-server","\u002Fsdks\u002Fserver","3.sdks\u002F5.server",[175,180,185,190,195],{"title":176,"path":177,"stem":178,"icon":179},"Node.js","\u002Fsdks\u002Fserver\u002Fnode-js","3.sdks\u002F5.server\u002F1.node-js","cd-logos-nodejs",{"title":181,"path":182,"stem":183,"icon":184},"Python","\u002Fsdks\u002Fserver\u002Fpython","3.sdks\u002F5.server\u002F2.python","cd-logos-python",{"title":186,"path":187,"stem":188,"icon":189},"Java","\u002Fsdks\u002Fserver\u002Fjava","3.sdks\u002F5.server\u002F3.java","cd-logos-java",{"title":191,"path":192,"stem":193,"icon":194},".NET","\u002Fsdks\u002Fserver\u002Fdotnet","3.sdks\u002F5.server\u002F4.dotnet","cd-logos-dotnet",{"title":196,"path":197,"stem":198,"icon":199},"Go","\u002Fsdks\u002Fserver\u002Fgo","3.sdks\u002F5.server\u002F5.go","cd-logos-go",{"title":201,"modifiedAt":73,"icon":202,"defaultOpen":12,"path":203,"stem":204,"children":205},"OpenFeature Providers","cd-logos-openfeature","\u002Fsdks\u002Fopenfeature","3.sdks\u002F6.openfeature\u002Findex",[206,207,212,216,220,224,228,232,236,240],{"title":55,"path":203,"stem":204,"icon":202},{"title":208,"path":209,"stem":210,"icon":202,"icons":211},"Web","\u002Fsdks\u002Fopenfeature\u002Fweb","3.sdks\u002F6.openfeature\u002F1.web",[202,117],{"title":144,"path":213,"stem":214,"icon":202,"icons":215},"\u002Fsdks\u002Fopenfeature\u002Fswift","3.sdks\u002F6.openfeature\u002F2.swift",[202,147],{"title":149,"path":217,"stem":218,"icon":202,"icons":219},"\u002Fsdks\u002Fopenfeature\u002Fandroid","3.sdks\u002F6.openfeature\u002F3.android",[202,152],{"title":139,"path":221,"stem":222,"icon":202,"icons":223},"\u002Fsdks\u002Fopenfeature\u002Fflutter","3.sdks\u002F6.openfeature\u002F4.flutter",[202,142],{"title":176,"path":225,"stem":226,"icon":202,"icons":227},"\u002Fsdks\u002Fopenfeature\u002Fnode-js","3.sdks\u002F6.openfeature\u002F5.node-js",[202,179],{"title":181,"path":229,"stem":230,"icon":202,"icons":231},"\u002Fsdks\u002Fopenfeature\u002Fpython","3.sdks\u002F6.openfeature\u002F6.python",[202,184],{"title":186,"path":233,"stem":234,"icon":202,"icons":235},"\u002Fsdks\u002Fopenfeature\u002Fjava","3.sdks\u002F6.openfeature\u002F7.java",[202,189],{"title":191,"path":237,"stem":238,"icon":202,"icons":239},"\u002Fsdks\u002Fopenfeature\u002Fdotnet","3.sdks\u002F6.openfeature\u002F8.dotnet",[202,194],{"title":196,"path":241,"stem":242,"icon":202,"icons":243},"\u002Fsdks\u002Fopenfeature\u002Fgo","3.sdks\u002F6.openfeature\u002F9.go",[202,199],{"title":245,"path":246,"stem":247,"icon":248},"Maven Repository","\u002Fsdks\u002Fmaven-repository","3.sdks\u002F7.maven-repository","i-lucide-package",{"title":250,"modifiedAt":73,"defaultOpen":9,"path":251,"stem":252,"children":253,"page":12},"Integrations","\u002Fintegrations","4.integrations",[254,259,264,269,274,278,283],{"title":255,"path":256,"stem":257,"icon":258},"MCP","\u002Fintegrations\u002Fmcp","4.integrations\u002F1.mcp","cd-logos-mcp",{"title":260,"path":261,"stem":262,"icon":263},"Agent Skills","\u002Fintegrations\u002Fagents","4.integrations\u002F2.agents","i-lucide-bot",{"title":265,"path":266,"stem":267,"icon":268},"CLI","\u002Fintegrations\u002Fcli","4.integrations\u002F3.cli","cd-logos-cli",{"title":270,"path":271,"stem":272,"icon":273},"Terraform","\u002Fintegrations\u002Fterraform","4.integrations\u002F4.terraform","cd-logos-terraform",{"title":275,"path":276,"stem":277,"icon":41},"Admin API","\u002Fintegrations\u002Fadmin-api","4.integrations\u002F5.admin-api",{"title":279,"path":280,"stem":281,"icon":282},"Webhooks","\u002Fintegrations\u002Fwebhooks","4.integrations\u002F6.webhooks","i-lucide-webhook",{"title":284,"modifiedAt":285,"icon":286,"defaultOpen":12,"path":287,"stem":288,"children":289},"Analytics","2026-11-08T17:10:01.000Z","i-lucide-chart-line","\u002Fintegrations\u002Fanalytics","4.integrations\u002F7.analytics\u002Findex",[290,292,296,300,304,308,312,316],{"title":55,"path":287,"stem":288,"icon":291},"i-lucide-blocks",{"title":293,"path":294,"stem":295,"icon":286},"Amplitude","\u002Fintegrations\u002Fanalytics\u002Famplitude","4.integrations\u002F7.analytics\u002F1.amplitude",{"title":297,"path":298,"stem":299,"icon":286},"Datadog","\u002Fintegrations\u002Fanalytics\u002Fdatadog","4.integrations\u002F7.analytics\u002F2.datadog",{"title":301,"path":302,"stem":303,"icon":286},"Google Analytics 4","\u002Fintegrations\u002Fanalytics\u002Fga4","4.integrations\u002F7.analytics\u002F3.ga4",{"title":305,"path":306,"stem":307,"icon":286},"Grafana","\u002Fintegrations\u002Fanalytics\u002Fgrafana","4.integrations\u002F7.analytics\u002F4.grafana",{"title":309,"path":310,"stem":311,"icon":286},"Mixpanel","\u002Fintegrations\u002Fanalytics\u002Fmixpanel","4.integrations\u002F7.analytics\u002F5.mixpanel",{"title":313,"path":314,"stem":315,"icon":286},"PostHog","\u002Fintegrations\u002Fanalytics\u002Fposthog","4.integrations\u002F7.analytics\u002F6.posthog",{"title":317,"path":318,"stem":319,"icon":286},"Segment","\u002Fintegrations\u002Fanalytics\u002Fsegment","4.integrations\u002F7.analytics\u002F7.segment",{"title":321,"modifiedAt":322,"defaultOpen":9,"path":323,"stem":324,"children":325,"page":12},"Migration Guides","2026-10-16T17:57:49.000Z","\u002Fmigration-guides","5.migration-guides",[326,331,335],{"title":327,"path":328,"stem":329,"icon":330},"From LaunchDarkly","\u002Fmigration-guides\u002Flaunchdarkly","5.migration-guides\u002F1.launchdarkly","i-lucide-baggage-claim",{"title":332,"path":333,"stem":334,"icon":330},"From Firebase Remote Config","\u002Fmigration-guides\u002Ffirebase-remote-config","5.migration-guides\u002F2.firebase-remote-config",{"title":336,"path":337,"stem":338,"icon":330},"From Firebase A\u002FB Testing","\u002Fmigration-guides\u002Ffirebase-ab-testing","5.migration-guides\u002F3.firebase-ab-testing",{"id":340,"title":341,"body":342,"description":1374,"extension":1375,"links":1376,"meta":1377,"navigation":1379,"path":58,"seo":1380,"stem":59,"__hash__":1383},"docs\u002F1.getting-started\u002F7.experiments\u002F1.how-results-are-computed.md","How ConfigDirector computes results",{"type":343,"value":344,"toc":1344},"minimark",[345,362,383,388,393,400,438,441,448,452,486,489,566,577,585,589,592,595,605,612,616,624,628,638,644,648,651,655,665,672,676,679,682,757,767,771,791,795,811,866,880,886,890,893,899,903,906,947,953,960,964,976,979,983,989,996,1000,1003,1017,1021,1053,1056,1059,1063,1077,1148,1151,1159,1222,1230,1234,1237,1251,1261,1265,1294,1297],[346,347,348,349,353,354,357,358,361],"p",{},"Once you attach metrics to an experiment, the ",[350,351,352],"strong",{},"Results"," panel on its ",[350,355,356],{},"Experiment"," tab compares each variation with the control. This page explains where every number on that panel comes from: what ConfigDirector counts, what it leaves out, how it compares the variations and what it does not do. It also explains how the ",[350,359,360],{},"How long to run"," card estimates the length of a run.",[346,363,364,365,369,370,374,375,378,379,382],{},"The examples follow ",[366,367,368],"code",{},"checkout-cta-copy"," from the ",[371,372,373],"a",{"href":51},"Experiments guide",", which compares two labels for a checkout button: ",[366,376,377],{},"control"," and ",[366,380,381],{},"treatment-a",".",[384,385,387],"h2",{"id":386},"what-is-counted","What is counted",[389,390,392],"h3",{"id":391},"exposures","Exposures",[346,394,395,396,399],{},"A context is ",[350,397,398],{},"exposed"," to a variation when the experiment's percentage rollout serves it that variation during a run. It counts once per variation, however often it evaluates the experiment, so results compare contexts rather than page views or evaluations.",[346,401,402,403,406,407,410,411,378,414,410,417,419,420,422,423,425,426,428,429,432,433,437],{},"ConfigDirector tells contexts apart by their context kind and ",[366,404,405],{},"id",". ",[366,408,409],{},"user"," ",[366,412,413],{},"42",[366,415,416],{},"device",[366,418,413],{}," are two contexts, and so are a person signed in as a ",[366,421,409],{}," and the same person's ",[366,424,416],{}," before signing in: ConfigDirector does not merge them. Contexts without an ",[366,427,405],{},", or marked ",[366,430,431],{},"anonymous",", are never counted. ",[371,434,436],{"href":435},"\u002Fgetting-started\u002Fexperiments#give-every-context-a-stable-identifier","Give every context a stable identifier"," explains what they are served instead.",[346,439,440],{},"Evaluations served by a conditional rule's single value or by the default targeting rule are not exposures, because those contexts were not assigned a variation at random.",[346,442,443,444,447],{},"Results are per percentage rollout. A run with two rollouts, such as one inside a conditional rule for ",[366,445,446],{},"plan is one of pro, team"," and one for everyone else, gets separate results for each, and they are never combined.",[389,449,451],{"id":450},"metrics","Metrics",[346,453,454,455,458,459,461,462,465,466,469,470,473,474,477,478,481,482,485],{},"A metric is an event your application reports with ",[366,456,457],{},"track",", defined once per project on the ",[350,460,451],{}," page. It has a direction, ",[350,463,464],{},"Higher is better"," or ",[350,467,468],{},"Lower is better",", and the metric kinds it reports. You attach metrics to an experiment on its ",[350,471,472],{},"Settings"," tab, each with a role: one ",[350,475,476],{},"primary"," metric and kind decide the experiment, ",[350,479,480],{},"secondary"," metrics are supporting evidence, and a ",[350,483,484],{},"guardrail"," is a metric that must not get worse.",[346,487,488],{},"Each metric kind turns the events of a variation's contexts into one number:",[490,491,492,508],"table",{},[493,494,495],"thead",{},[496,497,498,502,505],"tr",{},[499,500,501],"th",{},"Metric kind",[499,503,504],{},"What it measures",[499,506,507],{},"Example",[509,510,511,528,544],"tbody",{},[496,512,513,519,522],{},[514,515,516],"td",{},[350,517,518],{},"Conversion",[514,520,521],{},"The share of exposed contexts that reported the event at least once.",[514,523,524,527],{},[366,525,526],{},"checkout_completed",": the share of contexts that completed a checkout.",[496,529,530,535,538],{},[514,531,532],{},[350,533,534],{},"Count per context",[514,536,537],{},"How many times each exposed context reported the event, averaged over the exposed contexts.",[514,539,540,543],{},[366,541,542],{},"item_added_to_cart",": how many times each context added an item to its cart.",[496,545,546,551,558],{},[514,547,548],{},[350,549,550],{},"Value per context",[514,552,553,554,557],{},"The sum of the event's ",[366,555,556],{},"value"," per exposed context, averaged over the exposed contexts.",[514,559,560,562,563,565],{},[366,561,526],{}," reported with the order total as its ",[366,564,556],{},": revenue per context.",[346,567,568,569,571,572,574,575,382],{},"A context that reported nothing counts as zero in both averages, so they are averages over every exposed context, not only over the ones that reported the event. An event reported without a ",[366,570,556],{}," counts toward ",[350,573,534],{}," and adds nothing to ",[350,576,550],{},[346,578,579,580,584],{},"A metric's results cover the whole run, even when you attach it, or tick one of its kinds, after the run started. The one exception is a run that had already counted 100 other event names, the ",[371,581,583],{"href":582},"\u002Fmonitoring\u002Factivity#limits","most a run counts",", before you attached the metric: its event then counts from that moment on.",[389,586,588],{"id":587},"which-events-count","Which events count",[346,590,591],{},"An event counts toward a run when it happened while the run was open, at or after its context's first exposure in that run. From that first exposure to the end of the run, every event the context reports counts. There is no shorter window, and nothing after the run ends counts.",[346,593,594],{},"SDKs report evaluations in batches, about every 30 seconds, and ConfigDirector takes the start of the batch as the moment of exposure. So an event up to about 30 seconds before the exposure can count.",[346,596,597,598,601,602,604],{},"An event names no experiment. It counts toward every run open in its environment that exposed its context: when two experiments run at once in ",[366,599,600],{},"Production",", the ",[366,603,526],{}," of a context exposed to both counts in each.",[346,606,607,608,611],{},"If a context's exposure reaches ConfigDirector more than five minutes after its event, for example from a phone that was offline, the event may not be counted. Events dropped by a ",[371,609,610],{"href":582},"tracked event limit"," are never counted.",[384,613,615],{"id":614},"what-is-left-out","What is left out",[346,617,618,619,623],{},"Your team's own accounts, test accounts and bots are counted like any other context unless you keep them out of the split, as ",[371,620,622],{"href":621},"\u002Fgetting-started\u002Fexperiments#keep-internal-users-and-bots-out","Keep internal users and bots out"," shows.",[389,625,627],{"id":626},"crossovers","Crossovers",[346,629,630,631,634,635,637],{},"A ",[350,632,633],{},"crossover"," is a context exposed to more than one variation of the same rollout during a run. A context's share of the split depends only on its context kind and ",[366,636,405],{},", so it normally sees one variation for the whole run. A crossover happens when a variation's value is edited during a run.",[346,639,640,641,643],{},"Counting a crossover under each variation it saw would pull the variations' numbers toward each other. So ConfigDirector leaves it out of every metric, from every variation: its exposures, and every event it reported, before and after it crossed over. The panel says how many contexts crossed over, and each variation they saw counts them out, so the metrics count fewer contexts than the ",[350,642,392],{}," card.",[389,645,647],{"id":646},"variations-outside-the-split","Variations outside the split",[346,649,650],{},"A variation with a 0% share takes no part in the results, and neither does one that was not in the split when the run started.",[384,652,654],{"id":653},"the-control","The control",[346,656,657,658,660,661,664],{},"Every variation is compared with the rollout's ",[350,659,377],{},": the variation the default targeting rule served when the run started, if the split gives it a share. Otherwise it is the first variation in the split that has a share. That's why the ",[350,662,663],{},"Default variation"," you pick when you create an experiment is usually your control.",[346,666,667,668,671],{},"The control is fixed when the run starts, so changing the default targeting rule value during the run does not change it. It comes first in each table, tagged ",[350,669,670],{},"Control",", with no lift or chance to beat control of its own.",[384,673,675],{"id":674},"how-the-variations-are-compared","How the variations are compared",[346,677,678],{},"ConfigDirector uses a Bayesian method with flat priors. It starts with no opinion about any variation, so every number comes from your run's own data, and it answers the question a team asks directly: given what has happened so far, how likely is this variation to be better than the control, and by how much?",[346,680,681],{},"Each variation's row shows these numbers:",[683,684,685,707,725,731,751],"ul",{},[686,687,688,691,692,695,696,699,700,702,703,706],"li",{},[350,689,690],{},"The metric's value",", such as ",[366,693,694],{},"14.1%"," with ",[366,697,698],{},"176 of 1,246"," beneath it for ",[350,701,518],{},": the share or average described in ",[371,704,451],{"href":705},"#metrics",", over that variation's exposed contexts.",[686,708,709,712,713,716,717,720,721,724],{},[350,710,711],{},"Lift",": how much higher or lower the variation's value is than the control's, relative to the control's. A lift of ",[366,714,715],{},"+10%"," on a control conversion of ",[366,718,719],{},"12%"," means about ",[366,722,723],{},"13.2%",". For a metric where lower is better, a negative lift is the good direction.",[686,726,727,730],{},[350,728,729],{},"Likely range",": where the real lift very likely is, given the data so far. The real lift is as likely to be above the range as below it. A range that spans zero means the variation could still turn out better or worse than the control. The range narrows as the run gathers contexts.",[686,732,733,736,737,740,741,378,743,745,746,740,748,750],{},[350,734,735],{},"Chance to beat control",": the chance that the variation's real value is better than the control's, in the metric's direction. It answers whether the variation is better, not by how much: the lift and its range answer that. While no context has reported the event, it is ",[366,738,739],{},"50%"," for ",[350,742,534],{},[350,744,550],{},", and about ",[366,747,739],{},[350,749,518],{}," when the variations have similar exposures.",[686,752,753,756],{},[350,754,755],{},"Chance to be best",", with three or more variations: the chance that each variation, the control included, is the best of them all in the metric's direction. Across the variations, the chances add up to 100%.",[346,758,759,760,763,764,382],{},"The panel rounds the lift, its range and the chances to whole percentages, and shows a chance that rounds to 0% as ",[366,761,762],{},"\u003C1%"," and one that rounds to 100% as ",[366,765,766],{},">99%",[389,768,770],{"id":769},"the-details-behind-each-number","The details behind each number",[683,772,773,779,786],{},[686,774,775,776,778],{},"For ",[350,777,518],{},", the lift and its range compare the rates after counting one extra context that reported the event and one that did not in each variation. That is what starting with no opinion means. With hundreds of conversions it barely moves the lift away from the ratio of the rates shown; with only a few, it can move it a lot.",[686,780,775,781,378,783,785],{},[350,782,534],{},[350,784,550],{},", the lift compares the averages shown. It is left out when the control's average is zero or below, because there is nothing to take a relative change of, and the range is left out while the control's average is too uncertain to tell apart from zero. The chance to beat control and the verdict still show.",[686,787,788,790],{},[350,789,755],{}," is worked out by simulating the comparison many times. The same counts on the same day always give the same number, so reloading the page does not change it, and once a run has its final results, its chance to be best no longer changes.",[389,792,794],{"id":793},"a-worked-example","A worked example",[346,796,797,799,800,802,803,805,806,601,808,810],{},[366,798,368],{}," has one primary metric, Checkout, on the ",[366,801,526],{}," event, with ",[350,804,518],{}," as its primary kind. Partway through the run in ",[366,807,600],{},[350,809,352],{}," panel shows:",[490,812,813,830],{},[493,814,815],{},[496,816,817,820,822,825,827],{},[499,818,819],{},"Variation",[499,821,518],{},[499,823,824],{},"Lift · likely range",[499,826,735],{},[499,828,829],{},"Verdict",[509,831,832,848],{},[496,833,834,838,841,844,846],{},[514,835,836],{},[366,837,377],{},[514,839,840],{},"12.7% (147 of 1,154)",[514,842,843],{},"—",[514,845,843],{},[514,847,670],{},[496,849,850,854,857,860,863],{},[514,851,852],{},[366,853,381],{},[514,855,856],{},"14.1% (176 of 1,246)",[514,858,859],{},"+11% (−10% to +36%)",[514,861,862],{},"84%",[514,864,865],{},"No clear difference",[346,867,868,870,871,873,874,876,877,879],{},[366,869,381],{}," converts at 14.1% against the control's 12.7%, a lift of +11%. The likely range, −10% to +36%, still spans zero: the data so far allows ",[366,872,381],{}," to be somewhat worse than the control, or a lot better. Its chance to beat control is 84%. That sounds high, but it leaves about a one-in-six chance that ",[366,875,381],{}," is not better, which is not enough to call it likely better, so its verdict is ",[350,878,865],{},". With two variations, there is no chance to be best.",[346,881,882,883,382],{},"If the gap held as more contexts were exposed, the range would narrow and the chance to beat control would climb, until the verdict read ",[350,884,885],{},"Likely better",[384,887,889],{"id":888},"when-results-appear","When results appear",[346,891,892],{},"The panel shows each variation's exposures and value from the start of the run. The lift, the likely range, the chances and the verdict appear once every variation in the rollout has enough exposures, not counting crossovers. Until then, the panel says the run needs more exposures.",[346,894,895,896,898],{},"The minimum is the same for every metric, and high enough that the ",[350,897,392],{}," card can already check the split against your shares, so a result never shows while that check still needs more exposures. In a run with two rollouts, one can show results while the other is still waiting.",[384,900,902],{"id":901},"what-the-verdict-means","What the verdict means",[346,904,905],{},"Each variation gets a verdict for each metric kind, in the metric's direction:",[490,907,908,917],{},[493,909,910],{},[496,911,912,914],{},[499,913,829],{},[499,915,916],{},"What it means",[509,918,919,928,938],{},[496,920,921,925],{},[514,922,923],{},[350,924,885],{},[514,926,927],{},"The chance to beat control is high enough to say the variation is very likely better than the control.",[496,929,930,935],{},[514,931,932],{},[350,933,934],{},"Likely worse",[514,936,937],{},"The chance to beat control is low enough to say the variation is very likely worse than the control.",[496,939,940,944],{},[514,941,942],{},[350,943,865],{},[514,945,946],{},"The chance is in between: the data cannot yet tell the two apart, or there is no real difference to find.",[346,948,949,950,952],{},"The bar is high, the same in both directions and the same for every metric, and it cannot be changed. A ",[350,951,885],{}," verdict still leaves a small chance that the variation is not better.",[346,954,955,956,959],{},"A verdict informs a decision and never makes one: nothing ships or stops on its own, and your team decides. When you ship a variation, the ",[350,957,958],{},"Ship"," dialog shows the primary metric's verdict for it and warns you when a guardrail is likely worse.",[384,961,963],{"id":962},"how-long-a-run-needs","How long a run needs",[346,965,966,967,969,970,972,973,975],{},"The ",[350,968,360],{}," card on the ",[350,971,356],{}," tab estimates how many exposures a run needs before a variation has a good chance of reading ",[350,974,885],{}," on the primary metric, in the kind it decides on, and how many days that takes. It gives a row for each of five lifts: 2%, 5%, 10%, 20% and 50%, in the primary metric's better direction, so they read −2% to −50% when lower is better.",[346,977,978],{},"A row's lift is one a variation might really have, not the one measured so far. The smaller it is, the longer the run: half the lift takes about four times the exposures. Pick the smallest lift that would be worth shipping, and read its row.",[389,980,982],{"id":981},"before-a-run","Before a run",[346,984,985,986,988],{},"While no run is open in the environment, the card estimates from two numbers you type for ",[350,987,518],{},": the control's conversion today, and how many distinct contexts would reach the experiment each day. Each row gives the exposures needed, all variations together, and the days it takes to expose that many contexts. When the environment has an ended run with results, both numbers start from the most recent one: what its control converted, and how many contexts it exposed a day. You can change them, and nothing you type is saved.",[346,990,775,991,378,993,995],{},[350,992,534],{},[350,994,550],{},", the estimate also needs how much the number varies from one context to the next, which nobody knows before measuring it. So for those kinds the card estimates before a run only once an earlier run in the environment has results. It uses what that run's control averaged and how much it varied, and only the contexts a day can be typed.",[389,997,999],{"id":998},"while-a-run-is-open","While a run is open",[346,1001,1002],{},"During a run, the card estimates from the run's own numbers for each percentage rollout: the control's value so far, the run's shares, and the exposures so far, without the crossovers. Each row gives the exposures so far against those needed, and the days remaining at the run's recent pace: the new exposures a day over about the last 7 days, or since the run started when it is younger than that. The card says since when.",[346,1004,1005,1006,1009,1010,1012,1013,1016],{},"A row that says ",[350,1007,1008],{},"Enough exposures"," means the run is long enough for that lift: a variation that really is that much better has a good chance of reading ",[350,1011,885],{}," by now. The estimate appears once the variations can be compared, as ",[371,1014,889],{"href":1015},"#when-results-appear"," describes.",[389,1018,1020],{"id":1019},"what-the-estimate-assumes","What the estimate assumes",[683,1022,1023,1029,1041,1047],{},[686,1024,1025,1028],{},[350,1026,1027],{},"The control's numbers."," Before a run, they are what you type or what the last run measured. During a run, they are what the control has measured so far, which moves early in the run, and the estimate moves with it.",[686,1030,1031,1034,1035,1037,1038,1040],{},[350,1032,1033],{},"A good chance, not a certainty."," At the exposures a row gives, a variation whose real lift is that large has a good chance of reading ",[350,1036,885],{},", but it can still read ",[350,1039,865],{},". A variation whose real lift is smaller needs a longer run.",[686,1042,1043,1046],{},[350,1044,1045],{},"The pace holds."," The days assume contexts keep arriving at the pace you typed or the run measured. New contexts usually arrive more slowly as a run goes on, because the contexts that come back often were exposed in its first days. So take the days as the least the run needs.",[686,1048,1049,1052],{},[350,1050,1051],{},"The split."," Before a run, the card assumes an even split between the experiment's variations. The smallest share decides, so an uneven split, or another variation, needs more exposures.",[346,1054,1055],{},"A lift that would take the conversion to 100% or past it gets no row. During a run, while the control's conversion or average is zero, as when none of its contexts has reported the event yet, there is nothing to take a lift of, and the card says no estimate can be made.",[346,1057,1058],{},"Whatever the estimate says, run for at least one full week, so every day of the week is in the result.",[389,1060,1062],{"id":1061},"an-example","An example",[346,1064,1065,1066,1068,1069,1071,1072,378,1074,1076],{},"Before starting ",[366,1067,368],{}," in ",[366,1070,600],{},", the team expects the control to convert 12.7%, and about 400 contexts a day to reach the experiment, split 50\u002F50 between ",[366,1073,377],{},[366,1075,381],{},". The card reads:",[490,1078,1079,1092],{},[493,1080,1081],{},[496,1082,1083,1086,1089],{},[499,1084,1085],{},"For a real lift of",[499,1087,1088],{},"Exposures needed",[499,1090,1091],{},"Days",[509,1093,1094,1105,1116,1126,1137],{},[496,1095,1096,1099,1102],{},[514,1097,1098],{},"+2%",[514,1100,1101],{},"about 428,610",[514,1103,1104],{},"more than a year",[496,1106,1107,1110,1113],{},[514,1108,1109],{},"+5%",[514,1111,1112],{},"about 69,439",[514,1114,1115],{},"about 174 days",[496,1117,1118,1120,1123],{},[514,1119,715],{},[514,1121,1122],{},"about 17,714",[514,1124,1125],{},"about 45 days",[496,1127,1128,1131,1134],{},[514,1129,1130],{},"+20%",[514,1132,1133],{},"about 4,601",[514,1135,1136],{},"about 12 days",[496,1138,1139,1142,1145],{},[514,1140,1141],{},"+50%",[514,1143,1144],{},"about 813",[514,1146,1147],{},"about 3 days",[346,1149,1150],{},"A lift of +10% is the smallest the team would ship for, so they plan a run of about 45 days, rounded up to seven full weeks.",[346,1152,1153,1154,1158],{},"Partway through the run, at the point of the ",[371,1155,1157],{"href":1156},"#a-worked-example","worked example",", the run has 2,400 exposures, the control converts 147 of 1,154, and the run exposes about 400 new contexts a day. The card now reads the control's own numbers:",[490,1160,1161,1172],{},[493,1162,1163],{},[496,1164,1165,1167,1169],{},[499,1166,1085],{},[499,1168,392],{},[499,1170,1171],{},"Days remaining",[509,1173,1174,1183,1193,1203,1213],{},[496,1175,1176,1178,1181],{},[514,1177,1098],{},[514,1179,1180],{},"2,400 of about 427,131",[514,1182,1104],{},[496,1184,1185,1187,1190],{},[514,1186,1109],{},[514,1188,1189],{},"2,400 of about 69,199",[514,1191,1192],{},"about 167 days",[496,1194,1195,1197,1200],{},[514,1196,715],{},[514,1198,1199],{},"2,400 of about 17,653",[514,1201,1202],{},"about 39 days",[496,1204,1205,1207,1210],{},[514,1206,1130],{},[514,1208,1209],{},"2,400 of about 4,585",[514,1211,1212],{},"about 6 days",[496,1214,1215,1217,1220],{},[514,1216,1141],{},[514,1218,1219],{},"2,400 of about 810",[514,1221,1008],{},[346,1223,1224,1226,1227,1229],{},[366,1225,381],{}," reads ",[350,1228,865],{}," so far, with a lift of +11%. The run has enough exposures for a lift of +50%, but not yet for one of +10%, so the team keeps to its plan.",[384,1231,1233],{"id":1232},"how-fresh-the-numbers-are","How fresh the numbers are",[346,1235,1236],{},"While a run is going, its results can be up to 25 minutes behind what your applications report, and the panel's heading says so.",[346,1238,1239,1240,1243,1244,1247,1248,1250],{},"The daily chart of each metric kind adds a point when a day ends in UTC, with the value for the whole run up to that moment. Its last point holds the current numbers, the same as the table above it: a hollow point labeled ",[350,1241,1242],{},"Now"," while the run is going, and a point labeled ",[350,1245,1246],{},"Run ended"," once the run has ended. Hover over a day's point to see when that day ended in your own time zone, and over ",[350,1249,1246],{}," to see when the run ended.",[346,1252,1253,1254,1257,1258,1260],{},"When a run ends, its results keep taking in the last reports for up to 25 minutes. Then the panel shows the ",[350,1255,1256],{},"Final results",", as of the moment the run ended, and they no longer change: an event that arrives later is not added. A run that ended with no exposures, or before ConfigDirector had results, has no final results, and the panel shows its dates instead. The ",[350,1259,958],{}," dialog shows results that can be up to 25 minutes behind, so the final results can differ slightly from what you saw when you shipped.",[384,1262,1264],{"id":1263},"what-the-method-does-not-do","What the method does not do",[683,1266,1267,1277,1283],{},[686,1268,1269,1272,1273,382],{},[350,1270,1271],{},"No sequential test."," ConfigDirector does not ask you to fix the run's length in advance or adjust its numbers for how often you look. Its chances mean what they say each time you read them, but early in a run they can swing a lot. See ",[371,1274,1276],{"href":1275},"#reading-results-well","Reading results well",[686,1278,1279,1282],{},[350,1280,1281],{},"No variance reduction."," Results use only what happened during the run, not what each context did before it, so the ranges narrow only as the run gathers contexts.",[686,1284,1285,1288,1289,465,1291,1293],{},[350,1286,1287],{},"No correction for several metrics."," Each metric and kind is compared on its own. The more you attach, the more likely one of them reads ",[350,1290,885],{},[350,1292,934],{}," by chance alone. That's why the primary metric decides and secondary metrics only support it.",[384,1295,1276],{"id":1296},"reading-results-well",[683,1298,1299,1305,1315,1330],{},[686,1300,1301,1304],{},[350,1302,1303],{},"Pick the primary metric before you start the run."," Choosing it after you have seen the results makes it easy to pick whichever metric happens to look good.",[686,1306,1307,1310,1311,1314],{},[350,1308,1309],{},"Don't stop on the first Likely better day."," The chance to beat control can cross the line and fall back, especially early in a run. Decide before you start how long the run will go, and read the verdict then. ",[371,1312,963],{"href":1313},"#how-long-a-run-needs"," helps you size it, in full weeks so that every day of the week is in it.",[686,1316,1317,1320,1321,1323,1324,1326,1327,1329],{},[350,1318,1319],{},"Watch the guardrails."," A guardrail that reads ",[350,1322,934],{}," is a reason not to ship, even when the primary metric reads ",[350,1325,885],{},". The panel and the ",[350,1328,958],{}," dialog both warn you.",[686,1331,1332,1335,1336,1338,1339,1343],{},[350,1333,1334],{},"Check the split first."," When the ",[350,1337,392],{}," card says ",[371,1340,1342],{"href":1341},"\u002Fgetting-started\u002Fexperiments#the-mismatch-warning","the split does not match the configured shares",", the run's results are not reliable.",{"title":1345,"searchDepth":1346,"depth":1346,"links":1347},"",2,[1348,1354,1358,1359,1363,1364,1365,1371,1372,1373],{"id":386,"depth":1346,"text":387,"children":1349},[1350,1352,1353],{"id":391,"depth":1351,"text":392},3,{"id":450,"depth":1351,"text":451},{"id":587,"depth":1351,"text":588},{"id":614,"depth":1346,"text":615,"children":1355},[1356,1357],{"id":626,"depth":1351,"text":627},{"id":646,"depth":1351,"text":647},{"id":653,"depth":1346,"text":654},{"id":674,"depth":1346,"text":675,"children":1360},[1361,1362],{"id":769,"depth":1351,"text":770},{"id":793,"depth":1351,"text":794},{"id":888,"depth":1346,"text":889},{"id":901,"depth":1346,"text":902},{"id":962,"depth":1346,"text":963,"children":1366},[1367,1368,1369,1370],{"id":981,"depth":1351,"text":982},{"id":998,"depth":1351,"text":999},{"id":1019,"depth":1351,"text":1020},{"id":1061,"depth":1351,"text":1062},{"id":1232,"depth":1346,"text":1233},{"id":1263,"depth":1346,"text":1264},{"id":1296,"depth":1346,"text":1276},"What an experiment's results count and leave out, how each variation is compared with the control, what the lift, its likely range, the chances and the verdict mean, and how long a run needs","md",null,{"modifiedAt":1378},"2026-11-11T23:19:13.000Z",{"title":57,"icon":60},{"title":1381,"description":1382},"How ConfigDirector Computes Experiment Results","The method behind ConfigDirector's experiment results. What counts as an exposure, the three metric kinds, which events count, crossovers, the control, a Bayesian comparison with lift, likely range, chance to beat control and chance to be best, verdicts, how long a run needs, freshness, and what the method does not do.","uL1oRa-AcpEpcx3BQ8x4TyMUn2eoYUdXhGQSa8I_Nx4",[1385,1387],{"title":55,"path":51,"stem":52,"description":1386,"icon":8,"children":-1},"Run an A\u002FB test, from creating the experiment and its metrics to reading the results, shipping a variation and removing the config",{"title":62,"path":63,"stem":64,"description":1388,"icon":65,"children":-1},"Define a set of users once and use it in the targeting rules of every config in the project",1794464799407]