ggml : drop support for QK_K=64

[ggerganov]

Support only super-block size of 256

[github-actions]

📈 llama.cpp server for bench-server-baseline on Standard_NC4as_T4_v3 for phi-2-q4_0: 533 iterations 🚀

Expand details for performance related PR only

• Concurrent users: 8, duration: 10m
• HTTP request : avg=8765.81ms p(95)=22675.83ms fails=, finish reason: stop=483 truncated=50
• Prompt processing (pp): avg=104.94tk/s p(95)=449.68tk/s
• Token generation (tg): avg=38.75tk/s p(95)=49.16tk/s
• ggml-org/models/phi-2/ggml-model-q4_0.gguf parallel=8 ctx-size=16384 ngl=33 batch-size=2048 ubatch-size=256 pp=1024 pp+tg=2048 branch=gg/drop-qk_k-64 commit=a8dd83b5d8316823033ac961feb9a878700b57bc

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 533 iterations"
    y-axis "llamacpp:prompt_tokens_seconds"
    x-axis "llamacpp:prompt_tokens_seconds" 1716414680 --> 1716415306
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 299.61, 299.61, 299.61, 299.61, 299.61, 748.34, 748.34, 748.34, 748.34, 748.34, 800.86, 800.86, 800.86, 800.86, 800.86, 818.15, 818.15, 818.15, 818.15, 818.15, 890.56, 890.56, 890.56, 890.56, 890.56, 872.81, 872.81, 872.81, 872.81, 872.81, 872.63, 872.63, 872.63, 872.63, 872.63, 885.4, 885.4, 885.4, 885.4, 885.4, 897.48, 897.48, 897.48, 897.48, 897.48, 894.74, 894.74, 894.74, 894.74, 894.74, 915.45, 915.45, 915.45, 915.45, 915.45, 823.11, 823.11, 823.11, 823.11, 823.11, 834.06, 834.06, 834.06, 834.06, 834.06, 853.59, 853.59, 853.59, 853.59, 853.59, 839.24, 839.24, 839.24, 839.24, 839.24, 840.14, 840.14, 840.14, 840.14, 840.14, 840.95, 840.95, 840.95, 840.95, 840.95, 839.18, 839.18, 839.18, 839.18, 839.18, 841.4, 841.4, 841.4, 841.4, 841.4, 840.38, 840.38, 840.38, 840.38, 840.38, 846.88, 846.88, 846.88, 846.88, 846.88, 846.96, 846.96, 846.96, 846.96, 846.96, 849.91, 849.91, 849.91, 849.91, 849.91, 851.95, 851.95, 851.95, 851.95, 851.95, 850.65, 850.65, 850.65, 850.65, 850.65, 852.04, 852.04, 852.04, 852.04, 852.04, 860.63, 860.63, 860.63, 860.63, 860.63, 858.32, 858.32, 858.32, 858.32, 858.32, 856.23, 856.23, 856.23, 856.23, 856.23, 859.17, 859.17, 859.17, 859.17, 859.17, 861.38, 861.38, 861.38, 861.38, 861.38, 859.79, 859.79, 859.79, 859.79, 859.79, 859.71, 859.71, 859.71, 859.71, 859.71, 862.28, 862.28, 862.28, 862.28, 862.28, 867.46, 867.46, 867.46, 867.46, 867.46, 866.27, 866.27, 866.27, 866.27, 866.27, 866.46, 866.46, 866.46, 866.46, 866.46, 869.38, 869.38, 869.38, 869.38, 869.38, 867.08, 867.08, 867.08, 867.08, 867.08, 866.66, 866.66, 866.66, 866.66, 866.66, 869.12, 869.12, 869.12, 869.12, 869.12, 871.59, 871.59, 871.59, 871.59, 871.59, 881.29, 881.29, 881.29, 881.29, 881.29, 885.13, 885.13, 885.13, 885.13, 885.13, 884.32, 884.32, 884.32, 884.32, 884.32, 881.01, 881.01, 881.01, 881.01, 881.01, 878.56, 878.56, 878.56, 878.56, 878.56, 876.27, 876.27, 876.27, 876.27, 876.27, 879.29, 879.29, 879.29, 879.29, 879.29, 881.11, 881.11, 881.11, 881.11, 881.11, 880.17, 880.17, 880.17, 880.17, 880.17, 881.65, 881.65, 881.65, 881.65, 881.65, 881.9, 881.9, 881.9, 881.9, 881.9, 884.25, 884.25, 884.25, 884.25, 884.25, 884.06, 884.06, 884.06, 884.06, 884.06, 885.67, 885.67, 885.67, 885.67, 885.67, 889.36, 889.36, 889.36, 889.36, 889.36, 889.52, 889.52, 889.52, 889.52, 889.52, 889.11, 889.11, 889.11, 889.11, 889.11, 888.88, 888.88, 888.88, 888.88, 888.88, 890.45, 890.45, 890.45, 890.45]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 533 iterations"
    y-axis "llamacpp:predicted_tokens_seconds"
    x-axis "llamacpp:predicted_tokens_seconds" 1716414680 --> 1716415306
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 42.17, 42.17, 42.17, 42.17, 42.17, 42.21, 42.21, 42.21, 42.21, 42.21, 28.03, 28.03, 28.03, 28.03, 28.03, 29.45, 29.45, 29.45, 29.45, 29.45, 31.67, 31.67, 31.67, 31.67, 31.67, 32.33, 32.33, 32.33, 32.33, 32.33, 34.1, 34.1, 34.1, 34.1, 34.1, 34.77, 34.77, 34.77, 34.77, 34.77, 35.09, 35.09, 35.09, 35.09, 35.09, 35.35, 35.35, 35.35, 35.35, 35.35, 35.49, 35.49, 35.49, 35.49, 35.49, 35.05, 35.05, 35.05, 35.05, 35.05, 33.66, 33.66, 33.66, 33.66, 33.66, 33.33, 33.33, 33.33, 33.33, 33.33, 31.72, 31.72, 31.72, 31.72, 31.72, 30.1, 30.1, 30.1, 30.1, 30.1, 29.64, 29.64, 29.64, 29.64, 29.64, 29.83, 29.83, 29.83, 29.83, 29.83, 30.15, 30.15, 30.15, 30.15, 30.15, 30.03, 30.03, 30.03, 30.03, 30.03, 30.14, 30.14, 30.14, 30.14, 30.14, 30.13, 30.13, 30.13, 30.13, 30.13, 30.29, 30.29, 30.29, 30.29, 30.29, 30.2, 30.2, 30.2, 30.2, 30.2, 30.3, 30.3, 30.3, 30.3, 30.3, 30.71, 30.71, 30.71, 30.71, 30.71, 30.78, 30.78, 30.78, 30.78, 30.78, 30.55, 30.55, 30.55, 30.55, 30.55, 30.36, 30.36, 30.36, 30.36, 30.36, 30.4, 30.4, 30.4, 30.4, 30.4, 30.64, 30.64, 30.64, 30.64, 30.64, 30.75, 30.75, 30.75, 30.75, 30.75, 30.84, 30.84, 30.84, 30.84, 30.84, 31.0, 31.0, 31.0, 31.0, 31.0, 30.99, 30.99, 30.99, 30.99, 30.99, 30.7, 30.7, 30.7, 30.7, 30.7, 30.56, 30.56, 30.56, 30.56, 30.56, 30.09, 30.09, 30.09, 30.09, 30.09, 29.49, 29.49, 29.49, 29.49, 29.49, 29.5, 29.5, 29.5, 29.5, 29.5, 29.77, 29.77, 29.77, 29.77, 29.77, 29.86, 29.86, 29.86, 29.86, 29.86, 30.1, 30.1, 30.1, 30.1, 30.1, 30.03, 30.03, 30.03, 30.03, 30.03, 29.84, 29.84, 29.84, 29.84, 29.84, 29.47, 29.47, 29.47, 29.47, 29.47, 28.67, 28.67, 28.67, 28.67, 28.67, 28.15, 28.15, 28.15, 28.15, 28.15, 28.1, 28.1, 28.1, 28.1, 28.1, 28.08, 28.08, 28.08, 28.08, 28.08, 28.1, 28.1, 28.1, 28.1, 28.1, 28.23, 28.23, 28.23, 28.23, 28.23, 28.29, 28.29, 28.29, 28.29, 28.29, 28.34, 28.34, 28.34, 28.34, 28.34, 28.32, 28.32, 28.32, 28.32, 28.32, 28.34, 28.34, 28.34, 28.34, 28.34, 28.29, 28.29, 28.29, 28.29, 28.29, 28.38, 28.38, 28.38, 28.38, 28.38, 28.53, 28.53, 28.53, 28.53, 28.53, 28.67, 28.67, 28.67, 28.67, 28.67, 28.78, 28.78, 28.78, 28.78]
                    

Loading

Details

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 533 iterations"
    y-axis "llamacpp:kv_cache_usage_ratio"
    x-axis "llamacpp:kv_cache_usage_ratio" 1716414680 --> 1716415306
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.14, 0.14, 0.14, 0.14, 0.14, 0.3, 0.3, 0.3, 0.3, 0.3, 0.23, 0.23, 0.23, 0.23, 0.23, 0.14, 0.14, 0.14, 0.14, 0.14, 0.18, 0.18, 0.18, 0.18, 0.18, 0.14, 0.14, 0.14, 0.14, 0.14, 0.14, 0.14, 0.14, 0.14, 0.14, 0.11, 0.11, 0.11, 0.11, 0.11, 0.21, 0.21, 0.21, 0.21, 0.21, 0.16, 0.16, 0.16, 0.16, 0.16, 0.19, 0.19, 0.19, 0.19, 0.19, 0.31, 0.31, 0.31, 0.31, 0.31, 0.28, 0.28, 0.28, 0.28, 0.28, 0.42, 0.42, 0.42, 0.42, 0.42, 0.35, 0.35, 0.35, 0.35, 0.35, 0.38, 0.38, 0.38, 0.38, 0.38, 0.15, 0.15, 0.15, 0.15, 0.15, 0.12, 0.12, 0.12, 0.12, 0.12, 0.32, 0.32, 0.32, 0.32, 0.32, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.16, 0.16, 0.16, 0.16, 0.16, 0.17, 0.17, 0.17, 0.17, 0.17, 0.14, 0.14, 0.14, 0.14, 0.14, 0.18, 0.18, 0.18, 0.18, 0.18, 0.3, 0.3, 0.3, 0.3, 0.3, 0.21, 0.21, 0.21, 0.21, 0.21, 0.12, 0.12, 0.12, 0.12, 0.12, 0.14, 0.14, 0.14, 0.14, 0.14, 0.15, 0.15, 0.15, 0.15, 0.15, 0.19, 0.19, 0.19, 0.19, 0.19, 0.15, 0.15, 0.15, 0.15, 0.15, 0.14, 0.14, 0.14, 0.14, 0.14, 0.29, 0.29, 0.29, 0.29, 0.29, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.38, 0.38, 0.38, 0.38, 0.38, 0.25, 0.25, 0.25, 0.25, 0.25, 0.09, 0.09, 0.09, 0.09, 0.09, 0.13, 0.13, 0.13, 0.13, 0.13, 0.15, 0.15, 0.15, 0.15, 0.15, 0.31, 0.31, 0.31, 0.31, 0.31, 0.46, 0.46, 0.46, 0.46, 0.46, 0.62, 0.62, 0.62, 0.62, 0.62, 0.59, 0.59, 0.59, 0.59, 0.59, 0.52, 0.52, 0.52, 0.52, 0.52, 0.14, 0.14, 0.14, 0.14, 0.14, 0.25, 0.25, 0.25, 0.25, 0.25, 0.24, 0.24, 0.24, 0.24, 0.24, 0.2, 0.2, 0.2, 0.2, 0.2, 0.15, 0.15, 0.15, 0.15, 0.15, 0.19, 0.19, 0.19, 0.19, 0.19, 0.24, 0.24, 0.24, 0.24, 0.24, 0.16, 0.16, 0.16, 0.16, 0.16, 0.18, 0.18, 0.18, 0.18, 0.18, 0.12, 0.12, 0.12, 0.12, 0.12, 0.1, 0.1, 0.1, 0.1, 0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.1, 0.1, 0.1, 0.1, 0.1, 0.16, 0.16, 0.16, 0.16]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 533 iterations"
    y-axis "llamacpp:requests_processing"
    x-axis "llamacpp:requests_processing" 1716414680 --> 1716415306
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 8.0, 8.0, 8.0, 8.0, 8.0, 4.0, 4.0, 4.0, 4.0, 4.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 2.0, 2.0, 2.0, 2.0, 2.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 1.0, 1.0, 1.0, 1.0, 1.0, 6.0, 6.0, 6.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 3.0, 3.0, 3.0, 3.0, 6.0, 6.0, 6.0, 6.0, 6.0, 2.0, 2.0, 2.0, 2.0, 2.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 3.0, 3.0, 3.0, 3.0, 3.0, 7.0, 7.0, 7.0, 7.0, 7.0, 3.0, 3.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 3.0, 3.0, 3.0, 3.0, 3.0, 6.0, 6.0, 6.0, 6.0, 6.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 1.0, 1.0, 1.0, 1.0, 1.0, 3.0, 3.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 3.0, 3.0, 3.0, 3.0]
                    

Loading