{
  "_comment": "Measured llama.cpp text-generation throughput (tokens/sec). Three suites with different test conditions — do not mix rows across suites when comparing. Used to calibrate the tokens/sec estimator and rendered on /specs/tokens-per-second/.",
  "_source": [
    "https://github.com/XiongjieDai/GPU-Benchmarks-on-LLM-Inference",
    "https://github.com/ggml-org/llama.cpp/discussions/4167",
    "https://www.hardware-corner.net/rtx-5090-llm-benchmarks/"
  ],
  "_asof": "2026-07-13",
  "suites": {
    "llama3_q4km_tg1024": {
      "conditions": "llama.cpp, Llama 3 8B / 70B Q4_K_M, average tokens/sec over 1024 generated tokens",
      "source_index": 0,
      "rows": [
        { "hardware": "RTX 3070 8GB", "tg_8b": 70.9, "tg_70b": null },
        { "hardware": "RTX 4070 Ti 12GB", "tg_8b": 82.2, "tg_70b": null },
        { "hardware": "RTX 3080 10GB", "tg_8b": 106.4, "tg_70b": null },
        { "hardware": "RTX 4080 16GB", "tg_8b": 106.2, "tg_70b": null },
        { "hardware": "RTX 3090 24GB", "tg_8b": 111.7, "tg_70b": null },
        { "hardware": "RTX 4090 24GB", "tg_8b": 127.7, "tg_70b": null },
        { "hardware": "2x RTX 3090 48GB", "tg_8b": 108.1, "tg_70b": 16.3 },
        { "hardware": "2x RTX 4090 48GB", "tg_8b": 122.6, "tg_70b": 19.1 },
        { "hardware": "RTX 6000 Ada 48GB", "tg_8b": 131.0, "tg_70b": 18.4 },
        { "hardware": "A100 PCIe 80GB", "tg_8b": 138.3, "tg_70b": 22.1 },
        { "hardware": "H100 PCIe 80GB", "tg_8b": 144.5, "tg_70b": 25.0 },
        { "hardware": "Apple M1 Max 32c-GPU 64GB", "tg_8b": 34.5, "tg_70b": 4.1 },
        { "hardware": "Apple M3 Max 40c-GPU 64GB", "tg_8b": 50.7, "tg_70b": 7.5 },
        { "hardware": "Apple M2 Ultra 76c-GPU 192GB", "tg_8b": 76.3, "tg_70b": 12.1 }
      ]
    },
    "apple_7b_q4_0_tg128": {
      "conditions": "llama.cpp discussion #4167, LLaMA 7B v2 Q4_0, 128-token generation, ngl=99; ranges span multiple submissions",
      "source_index": 1,
      "rows": [
        { "chip": "M1", "tg_low": 14.2, "tg_high": 14.2 },
        { "chip": "M2", "tg_low": 21.9, "tg_high": 21.9 },
        { "chip": "M4", "tg_low": 24.1, "tg_high": 24.1 },
        { "chip": "M3 Pro", "tg_low": 30.7, "tg_high": 30.7 },
        { "chip": "M1 Pro", "tg_low": 35.5, "tg_high": 36.4 },
        { "chip": "M2 Pro", "tg_low": 37.9, "tg_high": 38.9 },
        { "chip": "M4 Pro", "tg_low": 49.6, "tg_high": 50.7 },
        { "chip": "M1 Max", "tg_low": 54.6, "tg_high": 61.2 },
        { "chip": "M3 Max", "tg_low": 56.6, "tg_high": 66.3 },
        { "chip": "M2 Max", "tg_low": 61.0, "tg_high": 66.0 },
        { "chip": "M4 Max", "tg_low": 70.0, "tg_high": 83.1 },
        { "chip": "M1 Ultra", "tg_low": 74.9, "tg_high": 83.7 },
        { "chip": "M3 Ultra", "tg_low": 88.4, "tg_high": 92.1 },
        { "chip": "M2 Ultra", "tg_low": 88.6, "tg_high": 94.3 }
      ]
    },
    "rtx5090_qwen3_q4kxl_4k": {
      "conditions": "llama.cpp on GeForce RTX 5090 32GB, Qwen3 family Q4_K_XL, 4k context, generation tokens/sec",
      "source_index": 2,
      "rows": [
        { "model": "Qwen3 8B", "tg": 185.9 },
        { "model": "Qwen3 14B", "tg": 123.8 },
        { "model": "Qwen3 30B-A3B MoE", "tg": 234.3 },
        { "model": "Qwen3 32B", "tg": 61.4 }
      ]
    }
  }
}