{
  "schema": 2,
  "betterbench_version": "0.6.0",
  "corpus_version": "1.0",
  "config": {
    "temperature": 0.7,
    "top_p": 0.95,
    "top_k": 20,
    "seed": 42,
    "greedy": false,
    "warmup": 1,
    "runs_per_category": 2,
    "unique_nonce": true,
    "run_single_stream": true,
    "concurrency_levels": [
      1,
      2,
      4,
      8
    ],
    "concurrency_requests": 8,
    "run_concurrency": true,
    "run_prefill": true,
    "prefill_depths": [
      2000,
      7000
    ],
    "prefill_runs": 2,
    "prefill_warmup": 1,
    "prefill_max_tokens": 16,
    "prefill_ctx_margin": 256,
    "max_model_len": 65536,
    "weights": {
      "code": 0.3,
      "reasoning": 0.2,
      "prose": 0.15,
      "json": 0.15,
      "file_edit": 0.1,
      "summarization": 0.1,
      "math": 0,
      "chat": 0
    },
    "conf": 0.95,
    "target_mde_pct": 1.0,
    "ab_max_pairs": 200,
    "ab_min_pairs": 12,
    "timeout_s": 180
  },
  "env": {
    "timestamp": "2026-09-17T07:23:50+0000",
    "host": "redacted",
    "os": "Linux x86_64",
    "python": "3.12.3",
    "endpoint": "redacted-local-endpoint",
    "model": "Qwen3.8-27B-Quark-AWQ-MXFP4",
    "gpu": {
      "vendor": "amd",
      "model": "Radeon AI PRO R9700",
      "count": 1,
      "architecture": "gfx1201",
      "vram_gb": 32
    },
    "categories": [
      "chat",
      "code",
      "file_edit",
      "json",
      "math",
      "prose",
      "reasoning",
      "summarization"
    ],
    "notes": {
      "scope": "64K default quick screen; historical thinking default differs",
      "thinking": "server default false; request has no chat_template_kwargs"
    },
    "corpus_hash": "e332dceff176d033",
    "max_model_len": 65536
  },
  "single_stream": {
    "chat": [
      {
        "ok": true,
        "category": "chat",
        "prompt_id": "chat-followup",
        "ttft_ms": 102.54816099768505,
        "update_gaps_ms": [
          43.7600800069049,
          43.63234399352223,
          43.31665503559634,
          43.390956008806825,
          43.369974999222904,
          43.501125008333474,
          43.542106985114515,
          43.401949980761856,
          43.742519977968186,
          43.60380704747513,
          43.63031598040834,
          43.474951991811395,
          43.60845300834626,
          43.436767999082804,
          43.453475984279066,
          43.68855699431151,
          43.527889996767044,
          43.86101203272119,
          43.45316701801494,
          43.71567798079923,
          43.46315300790593,
          43.529110960662365,
          43.80335402674973,
          43.5995269799605,
          43.48685703007504,
          43.5825299937278,
          43.47364901332185,
          43.4984479798004,
          43.6900649801828,
          43.6012830468826,
          43.57515397714451,
          43.57212601462379,
          43.53447398170829,
          43.73200098052621,
          43.53151499526575,
          43.53080503642559,
          43.639667972456664,
          43.68861101102084,
          43.562034028582275,
          43.6501219519414,
          43.67180203553289,
          43.463226000312716,
          43.67086797719821,
          43.952442007139325,
          43.57900301693007,
          43.577504984568805,
          43.686656979843974,
          43.63975900923833,
          43.705316027626395,
          43.642676959279925,
          43.55649102944881,
          43.664300988893956,
          43.75683603575453,
          43.842560960911214
        ],
        "tokens_per_update": 2.327272727272727,
        "chunking": "batched",
        "decode_tps": 53.94463291650152,
        "total_tps": 52.09853446269069,
        "pp_tps": 1121.424303285127,
        "prompt_tokens": 115,
        "completion_tokens": 128,
        "n_chunks": 55,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 2456.8829300114885,
        "error": null
      },
      {
        "ok": true,
        "category": "chat",
        "prompt_id": "chat-clarify",
        "ttft_ms": 99.95557903312147,
        "update_gaps_ms": [
          43.89121296117082,
          43.40653703548014,
          43.486137001309544,
          43.640556978061795,
          43.500540021341294,
          43.47097798017785,
          46.723467006813735,
          40.359922975767404,
          43.57399704167619,
          43.62981900339946,
          43.649839994031936,
          43.46709995297715,
          43.55885804397985,
          43.75087999505922,
          43.46126696327701,
          43.523752014152706,
          43.6823460040614,
          43.824009015224874,
          43.8937820144929,
          43.7020409735851,
          43.93057699780911,
          43.68657001759857,
          43.601610988844186,
          43.696423002984375,
          43.57497300952673,
          43.70525700505823,
          43.67839399492368,
          43.69915300048888,
          43.63496199948713,
          43.818270962219685,
          43.6344210174866,
          43.59030001796782,
          43.79360598977655,
          43.58992999186739,
          44.01064902776852
        ],
        "tokens_per_update": 3.5555555555555554,
        "chunking": "batched",
        "decode_tps": 83.12377088906099,
        "total_tps": 78.62934978496712,
        "pp_tps": 970.4310748663454,
        "prompt_tokens": 97,
        "completion_tokens": 128,
        "n_chunks": 36,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 1627.8908620006405,
        "error": null
      }
    ],
    "code": [
      {
        "ok": true,
        "category": "code",
        "prompt_id": "code-gen-medium",
        "ttft_ms": 105.149918992538,
        "update_gaps_ms": [
          43.81206200923771,
          43.62726799445227,
          43.58522203983739,
          43.5744869755581,
          43.59221598133445,
          43.62184804631397,
          43.75249898293987,
          43.44329598825425,
          43.81856299005449,
          43.5959940077737,
          43.8414309755899,
          43.56886900495738,
          43.86674804845825,
          43.822338979225606,
          43.833789008203894,
          43.776928971055895,
          43.769059993792325,
          43.62702404614538,
          43.58086199499667,
          43.84280100930482,
          44.30607799440622
        ],
        "tokens_per_update": 5.818181818181818,
        "chunking": "batched",
        "decode_tps": 138.30514783598542,
        "total_tps": 125.06379059155083,
        "pp_tps": 656.2059263678235,
        "prompt_tokens": 69,
        "completion_tokens": 128,
        "n_chunks": 22,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 1023.4776940196753,
        "error": null
      },
      {
        "ok": true,
        "category": "code",
        "prompt_id": "code-debug-medium",
        "ttft_ms": 105.99912801990286,
        "update_gaps_ms": [
          43.57272997731343,
          43.38697699131444,
          43.55511104222387,
          44.00906799128279,
          43.677988986019045,
          43.390184990130365,
          43.5499640298076,
          43.6245949822478,
          43.634478002786636,
          43.65734901512042,
          43.47528697689995,
          43.39264699956402,
          43.749636970460415,
          43.43432700261474,
          43.61774103017524,
          43.543125968426466,
          43.54500805493444,
          43.59379399102181,
          43.578415003139526,
          43.70702395681292,
          43.52000501239672,
          43.52157999528572,
          43.62150200176984,
          43.53676701430231,
          43.71236701263115,
          43.467725976370275,
          44.19706400949508,
          43.8779920223169,
          43.65675995359197,
          43.638284027110785,
          43.678645975887775,
          43.48721099086106,
          43.498118000570685,
          43.37142000440508,
          43.873538030311465,
          44.03569200076163
        ],
        "tokens_per_update": 3.4594594594594597,
        "chunking": "batched",
        "decode_tps": 80.87162422882076,
        "total_tps": 76.3514667131915,
        "pp_tps": 679.2508706909454,
        "prompt_tokens": 72,
        "completion_tokens": 128,
        "n_chunks": 37,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 1676.4576439745724,
        "error": null
      }
    ],
    "file_edit": [
      {
        "ok": true,
        "category": "file_edit",
        "prompt_id": "edit-refactor-medium",
        "ttft_ms": 101.62214504089206,
        "update_gaps_ms": [
          43.88108500279486,
          43.385967961512506,
          43.55057701468468,
          43.65947499172762,
          43.52330899564549,
          43.50756702478975,
          43.54997299378738,
          43.69649896398187,
          43.68604900082573,
          43.559292040299624,
          43.43777499161661,
          43.59765799017623,
          43.63577999174595,
          43.7065550358966,
          43.74258196912706,
          43.83600200526416,
          43.88088802807033,
          43.64887694828212,
          43.60244399867952,
          43.7218650477007,
          43.671468971297145,
          43.67073002504185,
          43.62603597110137,
          43.72118599712849,
          43.4903500135988,
          43.700037989765406,
          43.863036029506475,
          43.71588700450957,
          43.73234597733244,
          43.89542998978868,
          43.52569702314213,
          43.638239963911474,
          43.99962205206975
        ],
        "tokens_per_update": 3.764705882352941,
        "chunking": "batched",
        "decode_tps": 88.12955373571874,
        "total_tps": 82.96881018868406,
        "pp_tps": 1043.0797338252044,
        "prompt_tokens": 106,
        "completion_tokens": 128,
        "n_chunks": 34,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 1542.7484100218862,
        "error": null
      },
      {
        "ok": true,
        "category": "file_edit",
        "prompt_id": "edit-searchreplace-short",
        "ttft_ms": 106.59173200838268,
        "update_gaps_ms": [
          43.904501013457775,
          43.29698102083057,
          43.73973497422412,
          43.541934981476516,
          44.16934004984796
        ],
        "tokens_per_update": 5.666666666666667,
        "chunking": "batched",
        "decode_tps": 150.92441751813024,
        "total_tps": 104.5043254555521,
        "pp_tps": 694.23771061512,
        "prompt_tokens": 74,
        "completion_tokens": 34,
        "n_chunks": 6,
        "chunk_token_mismatch": true,
        "finish_reason": "stop",
        "ttfa_ms": 106.59173200838268,
        "reasoning_source": "none",
        "reasoning_chars": 0,
        "answer_chars": 98,
        "reasoning_chunks": 0,
        "answer_chunks": 6,
        "reasoning_tokens_est": 0,
        "answer_tokens_est": 34,
        "truncated_in_reasoning": false,
        "wall_ms": 325.3453850047663,
        "error": null
      }
    ],
    "json": [
      {
        "ok": true,
        "category": "json",
        "prompt_id": "json-extract-short",
        "ttft_ms": 106.99744202429429,
        "update_gaps_ms": [
          43.63086400553584,
          43.90596802113578,
          43.726493953727186,
          43.753370991908014,
          43.875686009414494,
          43.68908202741295,
          43.830031994730234
        ],
        "tokens_per_update": 7.125,
        "chunking": "batched",
        "decode_tps": 182.76076631450198,
        "total_tps": 124.6249003533765,
        "pp_tps": 700.9513366027095,
        "prompt_tokens": 75,
        "completion_tokens": 57,
        "n_chunks": 8,
        "chunk_token_mismatch": true,
        "finish_reason": "stop",
        "ttfa_ms": 106.99744202429429,
        "reasoning_source": "none",
        "reasoning_chars": 0,
        "answer_chars": 150,
        "reasoning_chunks": 0,
        "answer_chunks": 8,
        "reasoning_tokens_est": 0,
        "answer_tokens_est": 57,
        "truncated_in_reasoning": false,
        "wall_ms": 457.37248205114156,
        "error": null
      },
      {
        "ok": true,
        "category": "json",
        "prompt_id": "json-schema-medium",
        "ttft_ms": 106.86986398650333,
        "update_gaps_ms": [
          43.55163796572015,
          43.705720047000796,
          43.73419098556042,
          43.63225499400869,
          43.61552698537707,
          43.44682500232011,
          43.723601032979786,
          43.57915697619319,
          43.60678402008489,
          43.84267597924918,
          43.56792097678408,
          43.443341040983796,
          43.601737008430064,
          43.56756899505854,
          43.752224009949714,
          43.57789194909856,
          43.75355603406206,
          43.84042997844517,
          43.53908298071474,
          43.658805021550506,
          43.85224502766505,
          43.75254100887105,
          43.792543991003186,
          44.054253958165646
        ],
        "tokens_per_update": 5.12,
        "chunking": "batched",
        "decode_tps": 121.16094902906414,
        "total_tps": 110.80854789496415,
        "pp_tps": 711.1452860985964,
        "prompt_tokens": 76,
        "completion_tokens": 128,
        "n_chunks": 25,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 1155.1455409498885,
        "error": null
      }
    ],
    "math": [
      {
        "ok": true,
        "category": "math",
        "prompt_id": "math-word-medium",
        "ttft_ms": 106.95003595901653,
        "update_gaps_ms": [
          43.82989404257387,
          43.41161996126175,
          43.525206041522324,
          43.548073968850076,
          43.63435402046889,
          43.82341599557549,
          43.56276197358966,
          43.55336999287829,
          43.54575299657881,
          43.61711803358048,
          43.44275902258232,
          43.87748095905408,
          43.642395990900695,
          43.617619026917964,
          43.77029702300206,
          43.60875295242295,
          43.73628203757107,
          43.771411990746856,
          44.03913900023326,
          43.69937599403784,
          43.78603200893849,
          43.690156016964465,
          44.03271898627281
        ],
        "tokens_per_update": 5.333333333333333,
        "chunking": "batched",
        "decode_tps": 126.39759059537691,
        "total_tps": 115.12868839492111,
        "pp_tps": 691.911875825427,
        "prompt_tokens": 74,
        "completion_tokens": 128,
        "n_chunks": 24,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 1111.7993419757113,
        "error": null
      },
      {
        "ok": true,
        "category": "math",
        "prompt_id": "math-proof-medium",
        "ttft_ms": 97.93266304768622,
        "update_gaps_ms": [
          42.31411399086937,
          43.63964998628944,
          43.51955599850044,
          43.66744897561148,
          43.17809303756803,
          43.517564015928656,
          43.798111961223185,
          43.53851400082931,
          43.49555500084534,
          43.730486009735614,
          43.65717701148242,
          43.59236900927499,
          43.74363896204159,
          43.82917104521766,
          43.3414809522219,
          43.654475011862814,
          43.915031012147665,
          43.70447399560362,
          43.585942999925464,
          43.41019899584353,
          43.76120597589761,
          43.85445005027577,
          43.67406596429646,
          43.66174800088629,
          43.701334041543305,
          44.16361398762092
        ],
        "tokens_per_update": 4.7407407407407405,
        "chunking": "batched",
        "decode_tps": 112.02757409724134,
        "total_tps": 103.92315655846564,
        "pp_tps": 459.4994009106871,
        "prompt_tokens": 45,
        "completion_tokens": 128,
        "n_chunks": 27,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 1231.6792930359952,
        "error": null
      }
    ],
    "prose": [
      {
        "ok": true,
        "category": "prose",
        "prompt_id": "prose-story-long",
        "ttft_ms": 90.19838605308905,
        "update_gaps_ms": [
          43.3296249830164,
          43.640721996780485,
          43.26200200011954,
          43.409664009232074,
          43.362892989534885,
          43.699546018615365,
          43.50942897144705,
          43.644256016705185,
          43.747921008616686,
          43.40640897862613,
          43.48180798115209,
          43.44518203288317,
          43.70516899507493,
          43.67326997453347,
          43.68519701529294,
          43.50096400594339,
          43.63863501930609,
          43.49360498599708,
          43.648938997648656,
          43.64332900149748,
          43.61564200371504,
          43.69570000562817,
          43.505020963493735,
          43.51129702990875,
          43.654987995978445,
          43.589030974544585,
          43.83770003914833,
          43.515327968634665,
          43.442244990728796,
          43.65045600570738,
          43.755483988206834,
          43.62066905014217,
          43.55589795159176,
          43.44708001008257,
          43.60062401974574,
          43.638869014102966,
          43.71300700586289,
          43.84902899619192,
          43.87526598293334,
          43.64222998265177,
          43.71895198710263,
          43.6514780158177,
          44.00836903369054,
          43.75230998266488,
          43.552168004680425,
          43.68491598870605,
          43.78111701225862,
          43.65036799572408,
          43.714988976716995,
          43.73219900298864,
          43.64555201027542,
          43.907170998863876
        ],
        "tokens_per_update": 2.4150943396226414,
        "chunking": "batched",
        "decode_tps": 55.985519496874225,
        "total_tps": 54.26694856287138,
        "pp_tps": 598.6803352358946,
        "prompt_tokens": 54,
        "completion_tokens": 128,
        "n_chunks": 53,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 2358.7101060547866,
        "error": null
      },
      {
        "ok": true,
        "category": "prose",
        "prompt_id": "prose-continue-medium",
        "ttft_ms": 107.74143598973751,
        "update_gaps_ms": [
          43.91122004017234,
          43.54301397688687,
          43.767603987362236,
          43.51654299534857,
          43.6704340390861,
          43.947204016149044,
          43.55256794951856,
          43.519396043848246,
          43.66437898715958,
          43.62561699235812,
          43.63323497818783,
          43.62633504206315,
          43.593012960627675,
          43.70367800584063,
          43.81668701535091,
          43.45320799620822,
          43.73474599560723,
          43.71611401438713,
          43.49905700655654,
          43.80329098785296,
          43.54451800463721,
          43.529997987207025,
          43.65132801467553,
          43.660564988385886,
          43.67448599077761,
          43.71968103805557,
          43.69697597576305,
          43.64610998891294,
          43.60058403108269,
          43.72657497879118,
          43.7272529816255,
          43.618371011689305,
          43.62450202461332,
          43.784379959106445,
          43.795845995191485,
          43.705873016733676,
          43.64503303077072,
          43.60417398856953,
          43.6182229896076,
          43.714714993257076,
          43.86972601059824,
          43.73385198414326,
          43.66800800198689,
          43.57745102606714,
          43.78051595995203,
          43.85222704149783,
          43.57683495618403,
          43.70731604285538,
          43.579705990850925,
          43.675802007783204,
          44.08188600791618
        ],
        "tokens_per_update": 2.4615384615384617,
        "chunking": "batched",
        "decode_tps": 57.00973117243852,
        "total_tps": 54.80621787431277,
        "pp_tps": 751.7998925475184,
        "prompt_tokens": 81,
        "completion_tokens": 128,
        "n_chunks": 52,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 2335.5014260159805,
        "error": null
      }
    ],
    "reasoning": [
      {
        "ok": true,
        "category": "reasoning",
        "prompt_id": "reason-plan-medium",
        "ttft_ms": 113.54047001805156,
        "update_gaps_ms": [
          44.02620694600046,
          43.68444002466276,
          43.73723000753671,
          43.69007801869884,
          43.464964954182506,
          43.53251599241048,
          43.621350021567196,
          43.61464601242915,
          43.75025600893423,
          43.72359300032258,
          43.83386200061068,
          43.5740280081518,
          43.76397095620632,
          43.995681044179946,
          43.74368296703324,
          44.078089005779475,
          43.9282989827916,
          43.98788703838363,
          43.813071970362216,
          43.91700599808246,
          43.73406298691407,
          43.90061600133777,
          45.852792041841894,
          41.83421598281711,
          43.75321802217513,
          44.14021800039336,
          44.074048986658454
        ],
        "tokens_per_update": 4.571428571428571,
        "chunking": "batched",
        "decode_tps": 107.37505742745495,
        "total_tps": 98.73665691389226,
        "pp_tps": 836.7060659947607,
        "prompt_tokens": 95,
        "completion_tokens": 128,
        "n_chunks": 28,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 1296.37769801775,
        "error": null
      },
      {
        "ok": true,
        "category": "reasoning",
        "prompt_id": "reason-causal-medium",
        "ttft_ms": 105.8506679837592,
        "update_gaps_ms": [
          43.86629501823336,
          43.673990992829204,
          43.90104400226846,
          43.754831014666706,
          43.780218984466046,
          43.72137598693371,
          43.82086102850735,
          43.6892049619928,
          43.82195300422609,
          43.67440601345152,
          43.714308994822204,
          43.719336041249335,
          43.70301298331469,
          43.87256200425327,
          43.76855195732787,
          43.59897703398019,
          43.667134013958275,
          43.98847999982536,
          43.91162795946002,
          43.72046404751018,
          43.89718599850312,
          43.89839997747913,
          43.68006397271529,
          43.8886409974657,
          43.707039032597095,
          44.21064298367128,
          43.835613992996514,
          43.92104403814301,
          43.705613003112376,
          44.00756797986105
        ],
        "tokens_per_update": 4.129032258064516,
        "chunking": "batched",
        "decode_tps": 96.64258720832599,
        "total_tps": 90.13851175219753,
        "pp_tps": 623.5199196865386,
        "prompt_tokens": 66,
        "completion_tokens": 128,
        "n_chunks": 31,
        "chunk_token_mismatch": true,
        "finish_reason": "length",
        "ttfa_ms": null,
        "reasoning_source": "unknown",
        "reasoning_chars": null,
        "answer_chars": null,
        "reasoning_chunks": null,
        "answer_chunks": null,
        "reasoning_tokens_est": null,
        "answer_tokens_est": null,
        "truncated_in_reasoning": false,
        "wall_ms": 1420.0367580051534,
        "error": null
      }
    ],
    "summarization": [
      {
        "ok": true,
        "category": "summarization",
        "prompt_id": "summ-short-out",
        "ttft_ms": 103.7940540118143,
        "update_gaps_ms": [
          44.01767801027745,
          43.74740598723292,
          43.51968702394515,
          44.12652796600014,
          43.78378402907401,
          43.63318195100874,
          43.68960903957486,
          43.71742997318506,
          43.78465301124379,
          44.14670099504292,
          43.79383602645248,
          43.897338968236,
          43.83526399033144,
          43.75608201371506,
          43.82653400534764,
          43.86297700693831,
          43.76236698590219,
          43.785743007902056,
          44.02809002203867,
          44.01892295572907,
          43.80436101928353,
          44.23386603593826
        ],
        "tokens_per_update": 3.652173913043478,
        "chunking": "batched",
        "decode_tps": 86.03068554712766,
        "total_tps": 78.605109141117,
        "pp_tps": 1098.3288116583635,
        "prompt_tokens": 114,
        "completion_tokens": 84,
        "n_chunks": 23,
        "chunk_token_mismatch": true,
        "finish_reason": "stop",
        "ttfa_ms": 103.7940540118143,
        "reasoning_source": "none",
        "reasoning_chars": 0,
        "answer_chars": 418,
        "reasoning_chunks": 0,
        "answer_chunks": 23,
        "reasoning_tokens_est": 0,
        "answer_tokens_est": 84,
        "truncated_in_reasoning": false,
        "wall_ms": 1068.6328270239756,
        "error": null
      },
      {
        "ok": true,
        "category": "summarization",
        "prompt_id": "summ-tldr",
        "ttft_ms": 108.9286069618538,
        "update_gaps_ms": [
          43.98165998281911,
          43.50302304374054,
          43.64992497721687,
          43.74957398977131,
          43.75144501682371,
          43.51871402468532,
          43.69342897552997,
          43.718780972994864,
          43.89024502597749,
          43.482150009367615,
          44.42751198075712
        ],
        "tokens_per_update": 3.1666666666666665,
        "chunking": "batched",
        "decode_tps": 76.86451638893439,
        "total_tps": 64.36390620885201,
        "pp_tps": 761.9669645556575,
        "prompt_tokens": 83,
        "completion_tokens": 38,
        "n_chunks": 12,
        "chunk_token_mismatch": true,
        "finish_reason": "stop",
        "ttfa_ms": 108.9286069618538,
        "reasoning_source": "none",
        "reasoning_chars": 0,
        "answer_chars": 176,
        "reasoning_chunks": 0,
        "answer_chunks": 12,
        "reasoning_tokens_est": 0,
        "answer_tokens_est": 38,
        "truncated_in_reasoning": false,
        "wall_ms": 590.3930049971677,
        "error": null
      }
    ]
  },
  "prefill": [
    {
      "target_depth": 2000,
      "prompt_tokens": [
        1520,
        1490
      ],
      "ttft_ms": [
        435.0519660511054,
        430.1485880278051
      ],
      "pp_tps": [
        3493.83549233621,
        3463.919309444962
      ],
      "tokens_per_update": [],
      "batched_runs": 0
    },
    {
      "target_depth": 7000,
      "prompt_tokens": [
        5171,
        5217
      ],
      "ttft_ms": [
        1539.27707800176,
        1542.1330200042576
      ],
      "pp_tps": [
        3359.369196033781,
        3382.97665138225
      ],
      "tokens_per_update": [],
      "batched_runs": 0
    }
  ],
  "concurrency": [
    {
      "level": 1,
      "requests": 8,
      "ok": 8,
      "wall_s": 11.241560708032921,
      "aggregate_tps": 84.41888316467212,
      "ttft_ms": [
        105.86646100273356,
        101.9560860004276,
        103.02018700167537,
        106.70780198415741,
        107.32506198110059,
        107.51517699100077,
        108.71635598596185,
        103.20257698185742
      ],
      "decode_tps": [
        49.932449476016245,
        78.32479602290175,
        96.51188860049123,
        137.81631382920764,
        90.57152726212679,
        156.3598216620416,
        103.39486538068326,
        108.2022996688267
      ],
      "tokens_per_update": [
        2.169491525423729,
        3.3684210526315788,
        4.129032258064516,
        5.818181818181818,
        3.878787878787879,
        6.5,
        4.413793103448276,
        4.529411764705882
      ],
      "batched_runs": 8
    },
    {
      "level": 2,
      "requests": 8,
      "ok": 8,
      "wall_s": 6.549070871958975,
      "aggregate_tps": 152.69341553191612,
      "ttft_ms": [
        104.295555036515,
        228.08537603123114,
        166.59662395250052,
        165.39718897547573,
        168.64265903132036,
        169.09819497959688,
        169.08994497498497,
        167.4620530102402
      ],
      "decode_tps": [
        62.37110394370424,
        56.040610697672285,
        129.30962428243998,
        79.3507301813316,
        146.89978794512035,
        84.15715188946572,
        94.94326439516509,
        119.94052612082997
      ],
      "tokens_per_update": [
        2.9767441860465116,
        2.6666666666666665,
        5.818181818181818,
        4.0,
        6.5,
        4.266666666666667,
        4.571428571428571,
        5.333333333333333
      ],
      "batched_runs": 8
    },
    {
      "level": 4,
      "requests": 8,
      "ok": 8,
      "wall_s": 4.177516842959449,
      "aggregate_tps": 230.75909355689882,
      "ttft_ms": [
        107.9867429798469,
        288.44241105252877,
        182.52164899604395,
        289.1047170269303,
        289.3386189825833,
        183.2203819649294,
        178.69879596401006,
        180.29718101024628
      ],
      "decode_tps": [
        91.95675747052928,
        105.73464141234014,
        118.49806360541493,
        48.95787152926997,
        42.29509852178302,
        59.77400643655181,
        99.329754489413,
        80.36075550352972
      ],
      "tokens_per_update": [
        5.565217391304348,
        5.818181818181818,
        6.357142857142857,
        2.909090909090909,
        2.56,
        3.5555555555555554,
        5.095238095238095,
        4.413793103448276
      ],
      "batched_runs": 8
    },
    {
      "level": 8,
      "requests": 8,
      "ok": 8,
      "wall_s": 3.2291851229965687,
      "aggregate_tps": 304.1014877117788,
      "ttft_ms": [
        350.7426279829815,
        350.92992801219225,
        350.4667319939472,
        350.714870961383,
        351.5546119888313,
        350.97511496860534,
        351.94954404141754,
        114.95651700533926
      ],
      "decode_tps": [
        96.44428815017122,
        90.89395755450327,
        70.33842155176906,
        67.83088438261622,
        67.7987125532518,
        65.78851149663755,
        54.8306787043153,
        40.78879201823723
      ],
      "tokens_per_update": [
        6.357142857142857,
        6.095238095238095,
        4.7407407407407405,
        4.571428571428571,
        4.7407407407407405,
        4.413793103448276,
        3.4722222222222223,
        2.5098039215686274
      ],
      "batched_runs": 8
    }
  ],
  "sample_gate": {
    "rule": "n * min(pct, 100-pct) / 100 >= 5",
    "under_sampled": [
      {
        "section": "single_stream",
        "key": "chat",
        "metric": "ttft_p99",
        "n": 2,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "chat",
        "metric": "update_p99",
        "n": 89,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "code",
        "metric": "ttft_p99",
        "n": 2,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "code",
        "metric": "update_p99",
        "n": 57,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "file_edit",
        "metric": "ttft_p99",
        "n": 2,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "file_edit",
        "metric": "update_p99",
        "n": 38,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "json",
        "metric": "ttft_p99",
        "n": 2,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "json",
        "metric": "update_p99",
        "n": 31,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "math",
        "metric": "ttft_p99",
        "n": 2,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "math",
        "metric": "update_p99",
        "n": 49,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "prose",
        "metric": "ttft_p99",
        "n": 2,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "prose",
        "metric": "update_p99",
        "n": 103,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "reasoning",
        "metric": "ttft_p99",
        "n": 2,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "reasoning",
        "metric": "update_p99",
        "n": 57,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "summarization",
        "metric": "ttft_p99",
        "n": 2,
        "need": 500
      },
      {
        "section": "single_stream",
        "key": "summarization",
        "metric": "update_p99",
        "n": 33,
        "need": 500
      },
      {
        "section": "concurrency",
        "key": 1,
        "metric": "ttft_p99",
        "n": 8,
        "need": 500
      },
      {
        "section": "concurrency",
        "key": 2,
        "metric": "ttft_p99",
        "n": 8,
        "need": 500
      },
      {
        "section": "concurrency",
        "key": 4,
        "metric": "ttft_p99",
        "n": 8,
        "need": 500
      },
      {
        "section": "concurrency",
        "key": 8,
        "metric": "ttft_p99",
        "n": 8,
        "need": 500
      },
      {
        "section": "prefill",
        "key": 2000,
        "metric": "pp_p99",
        "n": 2,
        "need": 500
      },
      {
        "section": "prefill",
        "key": 7000,
        "metric": "pp_p99",
        "n": 2,
        "need": 500
      }
    ]
  }
}