{
  "$comment": "Benchmark data for /benchmarks/ and the homepage preview. Keep `runs` empty until a reproducible result set exists. Every metric value must be a number measured under `configuration`; use null for 'not measured' (rendered as '—', never as 0). See README.md › Publishing benchmark data.",
  "status": "pending",
  "message": "Reproducible performance results are being prepared.",
  "publishedAt": null,
  "configuration": [
    { "id": "hardware", "label": "Hardware", "hint": "CPU / GPU model, memory, power state", "value": null },
    { "id": "os", "label": "Operating system", "hint": "OS name and version", "value": null },
    { "id": "model", "label": "Model", "hint": "Model name and parameter count", "value": null },
    { "id": "quantization", "label": "Quantization", "hint": "e.g. Q4_K_M — same file for every engine", "value": null },
    { "id": "backend", "label": "Backend", "hint": "CPU (ISA), Metal, Vulkan or CUDA", "value": null },
    { "id": "engineVersion", "label": "DynaLM version", "hint": "Release tag or commit", "value": null },
    { "id": "baseline", "label": "Comparison baseline", "hint": "Engine, version and flags", "value": null },
    { "id": "harness", "label": "Load generator", "hint": "Tool, prompt set, output length", "value": null }
  ],
  "scenarios": [
    { "id": "single", "label": "Single user", "description": "One request at a time. Measures raw decode speed and latency without contention." },
    { "id": "multi", "label": "Multi-user", "description": "Several concurrent requests. Measures how scheduling and batching hold up under load." }
  ],
  "metrics": [
    { "id": "decode_tps", "label": "Decode throughput", "unit": "tok/s", "better": "higher", "scenario": "single", "description": "Output tokens generated per second for one stream, after the first token." },
    { "id": "prefill_tps", "label": "Prefill throughput", "unit": "tok/s", "better": "higher", "scenario": "single", "description": "Prompt tokens processed per second before generation starts." },
    { "id": "ttft_p50", "label": "Time to first token", "unit": "ms", "better": "lower", "scenario": "multi", "description": "Median delay from request arrival to the first streamed token." },
    { "id": "aggregate_tps", "label": "Multi-user throughput", "unit": "tok/s", "better": "higher", "scenario": "multi", "description": "Total output tokens per second across all concurrent requests." },
    { "id": "itl_p50", "label": "Inter-token latency P50", "unit": "ms", "better": "lower", "scenario": "multi", "description": "Median gap between consecutive tokens within a stream." },
    { "id": "itl_p95", "label": "Inter-token latency P95", "unit": "ms", "better": "lower", "scenario": "multi", "description": "95th-percentile gap between consecutive tokens." },
    { "id": "itl_p99", "label": "Inter-token latency P99", "unit": "ms", "better": "lower", "scenario": "multi", "description": "99th-percentile gap between consecutive tokens — the stutter users notice." },
    { "id": "memory_rss", "label": "Memory usage", "unit": "MB", "better": "lower", "scenario": "single", "description": "Peak resident memory of the server process, model loaded." },
    { "id": "failed", "label": "Failed requests and timeouts", "unit": "count", "better": "lower", "scenario": "multi", "description": "Requests that errored or exceeded the timeout during the run." }
  ],
  "runs": [],
  "reproduction": {
    "commands": [],
    "notes": "Exact commands, prompt sets and raw outputs will be published with the first result set."
  },
  "limitations": [
    "Results apply only to the listed hardware, model, quantization and backend.",
    "Comparisons use the same model file and comparable settings for every engine.",
    "Where a baseline engine is faster, the result is shown as measured."
  ]
}