{
  "gpu": "Tesla T4",
  "torch": "2.11.0+cu130",
  "transformers": "5.20.2",
  "cuda": "13.0",
  "python": "3.13.16",
  "date_utc": "2026-11-16 17:46",
  "decode_tokens": 64,
  "warmup_steps": 16,
  "repeats": 3,
  "rows": [
    {
      "model": "Qwen/Qwen2.5-0.5B",
      "precision": "fp16",
      "ctx": 1024,
      "weights_gb": 0.988,
      "step_ms_reps": [
        32.32,
        31.39,
        31.34
      ],
      "step_ms": 31.39,
      "tok_s": 31.9,
      "bytes_per_token_gb": 1.001,
      "tok_s_pred": 271.3,
      "ok": true,
      "measured_over_pred": 0.12
    },
    {
      "model": "Qwen/Qwen2.5-0.5B",
      "precision": "fp16",
      "ctx": 4096,
      "weights_gb": 0.988,
      "step_ms_reps": [
        31.79,
        31.68,
        32.33
      ],
      "step_ms": 31.79,
      "tok_s": 31.5,
      "bytes_per_token_gb": 1.039,
      "tok_s_pred": 261.4,
      "ok": true,
      "measured_over_pred": 0.12
    },
    {
      "model": "Qwen/Qwen2.5-1.5B",
      "precision": "fp16",
      "ctx": 1024,
      "weights_gb": 3.097,
      "step_ms_reps": [
        37.27,
        37.54,
        37.54
      ],
      "step_ms": 37.54,
      "tok_s": 26.6,
      "bytes_per_token_gb": 3.127,
      "tok_s_pred": 86.8,
      "ok": true,
      "measured_over_pred": 0.31
    },
    {
      "model": "Qwen/Qwen2.5-1.5B",
      "precision": "fp16",
      "ctx": 4096,
      "weights_gb": 3.097,
      "step_ms_reps": [
        44.65,
        44.83,
        44.98
      ],
      "step_ms": 44.83,
      "tok_s": 22.3,
      "bytes_per_token_gb": 3.215,
      "tok_s_pred": 84.5,
      "ok": true,
      "measured_over_pred": 0.26
    },
    {
      "model": "Qwen/Qwen2.5-3B",
      "precision": "fp16",
      "ctx": 1024,
      "weights_gb": 6.181,
      "step_ms_reps": [
        47.75,
        47.5,
        47.86
      ],
      "step_ms": 47.75,
      "tok_s": 20.9,
      "bytes_per_token_gb": 6.22,
      "tok_s_pred": 43.7,
      "ok": true,
      "measured_over_pred": 0.48
    },
    {
      "model": "Qwen/Qwen2.5-3B",
      "precision": "fp16",
      "ctx": 4096,
      "weights_gb": 6.181,
      "step_ms_reps": [
        76.69,
        77.21,
        84.17
      ],
      "step_ms": 77.21,
      "tok_s": 13.0,
      "bytes_per_token_gb": 6.334,
      "tok_s_pred": 42.9,
      "ok": true,
      "measured_over_pred": 0.3
    },
    {
      "model": "Qwen/Qwen2.5-1.5B",
      "precision": "int4",
      "ctx": 1024,
      "weights_gb": 1.224,
      "step_ms_reps": [
        51.01,
        50.98,
        51.03
      ],
      "step_ms": 51.01,
      "tok_s": 19.6,
      "bytes_per_token_gb": 1.254,
      "tok_s_pred": 216.6,
      "ok": true,
      "measured_over_pred": 0.09
    },
    {
      "model": "Qwen/Qwen2.5-1.5B",
      "precision": "int4",
      "ctx": 4096,
      "weights_gb": 1.224,
      "step_ms_reps": [
        51.39,
        50.88,
        50.59
      ],
      "step_ms": 50.88,
      "tok_s": 19.7,
      "bytes_per_token_gb": 1.342,
      "tok_s_pred": 202.4,
      "ok": true,
      "measured_over_pred": 0.1
    },
    {
      "model": "Qwen/Qwen2.5-7B",
      "precision": "int4",
      "ctx": 1024,
      "weights_gb": 5.862,
      "step_ms_reps": [
        62.01,
        63.1,
        64.49
      ],
      "step_ms": 63.1,
      "tok_s": 15.8,
      "bytes_per_token_gb": 5.923,
      "tok_s_pred": 45.9,
      "ok": true,
      "measured_over_pred": 0.34
    },
    {
      "model": "Qwen/Qwen2.5-7B",
      "precision": "int4",
      "ctx": 4096,
      "weights_gb": 5.862,
      "step_ms_reps": [
        115.98,
        109.06,
        106.9
      ],
      "step_ms": 109.06,
      "tok_s": 9.2,
      "bytes_per_token_gb": 6.099,
      "tok_s_pred": 44.5,
      "ok": true,
      "measured_over_pred": 0.21
    }
  ],
  "bitsandbytes": "0.50.2",
  "bandwidth": {
    "copy_gbs": 241.5,
    "read_gbs": 271.6
  }
}