{
  "date_utc": "2026-11-16 19:02",
  "python": "3.13.16",
  "decode_tokens": 64,
  "repeats": 3,
  "contexts": [
    1024,
    4096
  ],
  "rows": [
    {
      "stack": "transformers",
      "model": "Qwen2.5-0.5B-Instruct",
      "precision": "fp16",
      "ctx": 1024,
      "weights_gb": 0.988,
      "tok_s_reps": [
        32.1,
        32.3,
        32.5
      ],
      "tok_s": 32.3,
      "tok_s_pred": 271.6,
      "measured_over_pred": 0.12
    },
    {
      "stack": "transformers",
      "model": "Qwen2.5-0.5B-Instruct",
      "precision": "fp16",
      "ctx": 4096,
      "weights_gb": 0.988,
      "tok_s_reps": [
        31.7,
        31.7,
        31.4
      ],
      "tok_s": 31.7,
      "tok_s_pred": 261.7,
      "measured_over_pred": 0.12
    },
    {
      "stack": "transformers",
      "model": "Qwen2.5-1.5B-Instruct",
      "precision": "fp16",
      "ctx": 1024,
      "weights_gb": 3.097,
      "tok_s_reps": [
        26.4,
        26.6,
        26.8
      ],
      "tok_s": 26.6,
      "tok_s_pred": 86.9,
      "measured_over_pred": 0.31
    },
    {
      "stack": "transformers",
      "model": "Qwen2.5-1.5B-Instruct",
      "precision": "fp16",
      "ctx": 4096,
      "weights_gb": 3.097,
      "tok_s_reps": [
        22.7,
        22.6,
        22.6
      ],
      "tok_s": 22.6,
      "tok_s_pred": 84.6,
      "measured_over_pred": 0.27
    },
    {
      "stack": "transformers",
      "model": "Qwen2.5-3B-Instruct",
      "precision": "fp16",
      "ctx": 1024,
      "weights_gb": 6.181,
      "tok_s_reps": [
        21.1,
        20.9,
        21.1
      ],
      "tok_s": 21.1,
      "tok_s_pred": 43.7,
      "measured_over_pred": 0.48
    },
    {
      "stack": "transformers",
      "model": "Qwen2.5-3B-Instruct",
      "precision": "fp16",
      "ctx": 4096,
      "weights_gb": 6.181,
      "tok_s_reps": [
        13.3,
        13.3,
        13.2
      ],
      "tok_s": 13.3,
      "tok_s_pred": 42.9,
      "measured_over_pred": 0.31
    },
    {
      "stack": "transformers",
      "model": "Qwen2.5-1.5B-Instruct",
      "precision": "nf4",
      "ctx": 1024,
      "weights_gb": 1.224,
      "tok_s_reps": [
        19.7,
        19.4,
        19.8
      ],
      "tok_s": 19.7,
      "tok_s_pred": 216.8,
      "measured_over_pred": 0.09
    },
    {
      "stack": "transformers",
      "model": "Qwen2.5-1.5B-Instruct",
      "precision": "nf4",
      "ctx": 4096,
      "weights_gb": 1.224,
      "tok_s_reps": [
        19.3,
        19.4,
        19.8
      ],
      "tok_s": 19.4,
      "tok_s_pred": 202.6,
      "measured_over_pred": 0.1
    },
    {
      "stack": "transformers",
      "model": "Qwen2.5-7B-Instruct",
      "precision": "nf4",
      "ctx": 1024,
      "weights_gb": 5.862,
      "tok_s_reps": [
        17.7,
        17.4,
        17.1
      ],
      "tok_s": 17.4,
      "tok_s_pred": 45.9,
      "measured_over_pred": 0.38
    },
    {
      "stack": "transformers",
      "model": "Qwen2.5-7B-Instruct",
      "precision": "nf4",
      "ctx": 4096,
      "weights_gb": 5.862,
      "tok_s_reps": [
        9.5,
        9.4,
        9.7
      ],
      "tok_s": 9.5,
      "tok_s_pred": 44.6,
      "measured_over_pred": 0.21
    },
    {
      "stack": "vllm",
      "model": "Qwen2.5-0.5B-Instruct",
      "precision": "fp16",
      "ctx": 1024,
      "weights_gb": 0.988,
      "tok_s_reps": [
        161.3,
        162.3,
        162.5
      ],
      "tok_s": 162.3,
      "tok_s_pred": 271.6,
      "measured_over_pred": 0.6
    },
    {
      "stack": "vllm",
      "model": "Qwen2.5-0.5B-Instruct",
      "precision": "fp16",
      "ctx": 4096,
      "weights_gb": 0.988,
      "tok_s_reps": [
        132.3,
        132.4,
        129.6
      ],
      "tok_s": 132.3,
      "tok_s_pred": 261.8,
      "measured_over_pred": 0.51
    },
    {
      "stack": "vllm",
      "model": "Qwen2.5-1.5B-Instruct",
      "precision": "fp16",
      "ctx": 1024,
      "weights_gb": 3.097,
      "tok_s_reps": [
        57.8,
        56.6,
        54.5
      ],
      "tok_s": 56.6,
      "tok_s_pred": 86.9,
      "measured_over_pred": 0.65
    },
    {
      "stack": "vllm",
      "model": "Qwen2.5-1.5B-Instruct",
      "precision": "fp16",
      "ctx": 4096,
      "weights_gb": 3.097,
      "tok_s_reps": [
        48.5,
        49.8,
        52.9
      ],
      "tok_s": 49.8,
      "tok_s_pred": 84.6,
      "measured_over_pred": 0.59
    },
    {
      "stack": "vllm",
      "model": "Qwen2.5-3B-Instruct",
      "precision": "fp16",
      "ctx": 1024,
      "weights_gb": 6.181,
      "tok_s_reps": [
        30.5,
        29.1,
        27.4
      ],
      "tok_s": 29.1,
      "tok_s_pred": 43.7,
      "measured_over_pred": 0.67
    },
    {
      "stack": "vllm",
      "model": "Qwen2.5-3B-Instruct",
      "precision": "fp16",
      "ctx": 4096,
      "weights_gb": 6.181,
      "tok_s_reps": [
        26.9,
        30.4,
        30.5
      ],
      "tok_s": 30.4,
      "tok_s_pred": 42.9,
      "measured_over_pred": 0.71
    }
  ],
  "stacks": {
    "transformers": {
      "version": "5.20.2",
      "status": "ok",
      "bitsandbytes": "0.50.2"
    },
    "llamacpp": {
      "status": "failed",
      "nvcc": "/usr/local/cuda/bin/nvcc",
      "commit": "ad5cbfa",
      "configure_rc": 1,
      "configure_stderr": "CMAKE_BUILD_TYPE=Release\nCMake Error at ggml/src/ggml-cuda/CMakeLists.txt:182 (target_link_libraries):\n  Target \"ggml-cuda\" links to:\n\n    CUDA::cuda_driver\n\n  but the target was not found.  Possible reasons include:\n\n    * There is a typo in the target name.\n    * A find_package call is missing for an IMPORTED target.\n    * An ALIAS target is missing.\n\n\n\nCMake Generate step failed.  Build files cannot be regenerated correctly.\n",
      "build_seconds": 1677,
      "error": "build stderr: /usr/bin/ld: cannot find -lCUDA::cuda_driver: No such file or directory\ncollect2: error: ld returned 1 exit status\ngmake[3]: *** [ggml/src/ggml-cuda/CMakeFiles/ggml-cuda.dir/build.make:2627: bin/libggml-cuda.so.1.0.0] Error 1\ngmake[2]: *** [CMakeFiles/Makefile2:2609: ggml/src/ggml-cuda/CMakeFiles/ggml-cuda.dir/all] Error 2\ngmake[1]: *** [CMakeFiles/Makefile2:6082: tools/llama-bench/CMakeFiles/llama-bench.dir/rule] Error 2\ngmake: *** [Makefile:1791: llama-bench] Error 2\n | stdout: ttn-vec-instance-f16-f16.cu.o\n[ 62%] Building CUDA object ggml/src/ggml-cuda/CMakeFiles/ggml-cuda.dir/template-instances/fattn-vec-instance-q4_0-q4_0.cu.o\n[ 62%] Building CUDA object ggml/src/ggml-cuda/CMakeFiles/ggml-cuda.dir/template-instances/fattn-vec-instance-q8_0-q8_0.cu.o\n[ 62%] Building CUDA object ggml/src/ggml-cuda/CMakeFiles/ggml-cuda.dir/template-instances/fattn-vec-instance-bf16-bf16.cu.o\n[ 62%] Building CUDA object ggml/src/ggml-cuda/CMakeFiles/ggml-cuda.dir/template-instances/fattn-vec-instance-f8_e4m3-f8_e4m3.cu.o\n[ 62%] Linking CUDA shared library ../../../bin/libggml-cuda.so\n"
    },
    "vllm": {
      "status": "ok",
      "install_seconds": 212,
      "version": "0.33.0"
    }
  },
  "gpu": "Tesla T4",
  "torch": "2.11.0+cu130",
  "cuda_torch": "13.0",
  "driver": "580.178.04\n580.178.04",
  "read_bandwidth_gbs": 271.9
}