llama : fix restoring the number of outputs from state files

[compilade]

Should fix #6679

[github-actions]

📈 llama.cpp server for bench-server-baseline on Standard_NC4as_T4_v3 for phi-2-q4_0: 426 iterations 🚀

Expand details for performance related PR only

• Concurrent users: 8, duration: 10m
• HTTP request : avg=11084.74ms p(95)=28142.11ms fails=, finish reason: stop=372 truncated=54
• Prompt processing (pp): avg=120.88tk/s p(95)=542.18tk/s
• Token generation (tg): avg=22.89tk/s p(95)=35.53tk/s
• ggml-org/models/phi-2/ggml-model-q4_0.gguf parallel=8 ctx-size=16384 ngl=33 batch-size=2048 ubatch-size=256 pp=1024 pp+tg=2048 branch=compilade/fix-prompt-cache commit=5598afb778ab7af886711bb9a44bebf9acbe4119

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 426 iterations"
    y-axis "llamacpp:prompt_tokens_seconds"
    x-axis "llamacpp:prompt_tokens_seconds" 1713187143 --> 1713187777
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 657.31, 657.31, 657.31, 657.31, 657.31, 649.33, 649.33, 649.33, 649.33, 649.33, 642.01, 642.01, 642.01, 642.01, 642.01, 656.58, 656.58, 656.58, 656.58, 656.58, 675.73, 675.73, 675.73, 675.73, 675.73, 697.63, 697.63, 697.63, 697.63, 697.63, 678.08, 678.08, 678.08, 678.08, 678.08, 675.81, 675.81, 675.81, 675.81, 675.81, 685.47, 685.47, 685.47, 685.47, 685.47, 702.97, 702.97, 702.97, 702.97, 702.97, 702.69, 702.69, 702.69, 702.69, 702.69, 710.31, 710.31, 710.31, 710.31, 710.31, 705.74, 705.74, 705.74, 705.74, 705.74, 723.41, 723.41, 723.41, 723.41, 723.41, 735.73, 735.73, 735.73, 735.73, 735.73, 729.78, 729.78, 729.78, 729.78, 729.78, 738.23, 738.23, 738.23, 738.23, 738.23, 622.6, 622.6, 622.6, 622.6, 622.6, 599.24, 599.24, 599.24, 599.24, 599.24, 607.57, 607.57, 607.57, 607.57, 607.57, 607.78, 607.78, 607.78, 607.78, 607.78, 607.34, 607.34, 607.34, 607.34, 607.34, 612.18, 612.18, 612.18, 612.18, 612.18, 613.77, 613.77, 613.77, 613.77, 613.77, 614.1, 614.1, 614.1, 614.1, 614.1, 613.5, 613.5, 613.5, 613.5, 613.5, 618.47, 618.47, 618.47, 618.47, 618.47, 618.87, 618.87, 618.87, 618.87, 618.87, 619.51, 619.51, 619.51, 619.51, 619.51, 590.55, 590.55, 590.55, 590.55, 590.55, 590.11, 590.11, 590.11, 590.11, 590.11, 594.27, 594.27, 594.27, 594.27, 594.27, 596.54, 596.54, 596.54, 596.54, 596.54, 593.82, 593.82, 593.82, 593.82, 593.82, 592.32, 592.32, 592.32, 592.32, 592.32, 594.33, 594.33, 594.33, 594.33, 594.33, 596.42, 596.42, 596.42, 596.42, 596.42, 601.57, 601.57, 601.57, 601.57, 601.57, 602.71, 602.71, 602.71, 602.71, 602.71, 602.62, 602.62, 602.62, 602.62, 602.62, 606.34, 606.34, 606.34, 606.34, 606.34, 610.47, 610.47, 610.47, 610.47, 610.47, 611.86, 611.86, 611.86, 611.86, 611.86, 616.28, 616.28, 616.28, 616.28, 616.28, 621.95, 621.95, 621.95, 621.95, 621.95, 623.4, 623.4, 623.4, 623.4, 623.4, 619.86, 619.86, 619.86, 619.86, 619.86, 619.57, 619.57, 619.57, 619.57, 619.57, 619.29, 619.29, 619.29, 619.29, 619.29, 620.59, 620.59, 620.59, 620.59, 620.59, 617.93, 617.93, 617.93, 617.93, 617.93, 618.87, 618.87, 618.87, 618.87, 618.87, 626.03, 626.03, 626.03, 626.03, 626.03, 604.2, 604.2, 604.2, 604.2, 604.2, 603.82, 603.82, 603.82, 603.82, 603.82, 603.26, 603.26, 603.26, 603.26, 603.26, 602.33, 602.33, 602.33, 602.33, 602.33, 601.43, 601.43, 601.43, 601.43, 601.43, 601.87, 601.87, 601.87, 601.87, 601.87, 605.52, 605.52, 605.52, 605.52, 605.52, 605.16, 605.16, 605.16]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 426 iterations"
    y-axis "llamacpp:predicted_tokens_seconds"
    x-axis "llamacpp:predicted_tokens_seconds" 1713187143 --> 1713187777
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 36.86, 36.86, 36.86, 36.86, 36.86, 24.8, 24.8, 24.8, 24.8, 24.8, 21.77, 21.77, 21.77, 21.77, 21.77, 21.59, 21.59, 21.59, 21.59, 21.59, 21.65, 21.65, 21.65, 21.65, 21.65, 21.97, 21.97, 21.97, 21.97, 21.97, 21.78, 21.78, 21.78, 21.78, 21.78, 22.38, 22.38, 22.38, 22.38, 22.38, 23.33, 23.33, 23.33, 23.33, 23.33, 23.56, 23.56, 23.56, 23.56, 23.56, 23.78, 23.78, 23.78, 23.78, 23.78, 23.83, 23.83, 23.83, 23.83, 23.83, 23.7, 23.7, 23.7, 23.7, 23.7, 23.82, 23.82, 23.82, 23.82, 23.82, 23.82, 23.82, 23.82, 23.82, 23.82, 23.46, 23.46, 23.46, 23.46, 23.46, 23.15, 23.15, 23.15, 23.15, 23.15, 23.15, 23.15, 23.15, 23.15, 23.15, 22.47, 22.47, 22.47, 22.47, 22.47, 22.39, 22.39, 22.39, 22.39, 22.39, 22.46, 22.46, 22.46, 22.46, 22.46, 22.53, 22.53, 22.53, 22.53, 22.53, 22.53, 22.53, 22.53, 22.53, 22.53, 22.29, 22.29, 22.29, 22.29, 22.29, 22.21, 22.21, 22.21, 22.21, 22.21, 21.96, 21.96, 21.96, 21.96, 21.96, 21.85, 21.85, 21.85, 21.85, 21.85, 21.66, 21.66, 21.66, 21.66, 21.66, 21.7, 21.7, 21.7, 21.7, 21.7, 21.85, 21.85, 21.85, 21.85, 21.85, 21.87, 21.87, 21.87, 21.87, 21.87, 21.93, 21.93, 21.93, 21.93, 21.93, 22.0, 22.0, 22.0, 22.0, 22.0, 22.06, 22.06, 22.06, 22.06, 22.06, 21.92, 21.92, 21.92, 21.92, 21.92, 21.95, 21.95, 21.95, 21.95, 21.95, 22.13, 22.13, 22.13, 22.13, 22.13, 22.24, 22.24, 22.24, 22.24, 22.24, 22.38, 22.38, 22.38, 22.38, 22.38, 22.39, 22.39, 22.39, 22.39, 22.39, 22.5, 22.5, 22.5, 22.5, 22.5, 22.55, 22.55, 22.55, 22.55, 22.55, 22.55, 22.55, 22.55, 22.55, 22.55, 22.54, 22.54, 22.54, 22.54, 22.54, 22.45, 22.45, 22.45, 22.45, 22.45, 22.38, 22.38, 22.38, 22.38, 22.38, 22.14, 22.14, 22.14, 22.14, 22.14, 22.15, 22.15, 22.15, 22.15, 22.15, 22.14, 22.14, 22.14, 22.14, 22.14, 22.18, 22.18, 22.18, 22.18, 22.18, 22.32, 22.32, 22.32, 22.32, 22.32, 22.37, 22.37, 22.37, 22.37, 22.37, 22.4, 22.4, 22.4, 22.4, 22.4, 22.32, 22.32, 22.32, 22.32, 22.32, 22.08, 22.08, 22.08, 22.08, 22.08, 21.93, 21.93, 21.93, 21.93, 21.93, 21.84, 21.84, 21.84, 21.84, 21.84, 21.79, 21.79, 21.79, 21.79, 21.79, 20.85, 20.85, 20.85, 20.85, 20.85, 20.77, 20.77, 20.77, 20.77, 20.77, 20.85, 20.85, 20.85]
                    

Loading

Details

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 426 iterations"
    y-axis "llamacpp:kv_cache_usage_ratio"
    x-axis "llamacpp:kv_cache_usage_ratio" 1713187143 --> 1713187777
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.29, 0.29, 0.29, 0.29, 0.29, 0.29, 0.29, 0.29, 0.29, 0.29, 0.19, 0.19, 0.19, 0.19, 0.19, 0.21, 0.21, 0.21, 0.21, 0.21, 0.19, 0.19, 0.19, 0.19, 0.19, 0.22, 0.22, 0.22, 0.22, 0.22, 0.15, 0.15, 0.15, 0.15, 0.15, 0.19, 0.19, 0.19, 0.19, 0.19, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.14, 0.14, 0.14, 0.14, 0.14, 0.22, 0.22, 0.22, 0.22, 0.22, 0.11, 0.11, 0.11, 0.11, 0.11, 0.18, 0.18, 0.18, 0.18, 0.18, 0.19, 0.19, 0.19, 0.19, 0.19, 0.16, 0.16, 0.16, 0.16, 0.16, 0.24, 0.24, 0.24, 0.24, 0.24, 0.35, 0.35, 0.35, 0.35, 0.35, 0.13, 0.13, 0.13, 0.13, 0.13, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.12, 0.12, 0.12, 0.12, 0.12, 0.24, 0.24, 0.24, 0.24, 0.24, 0.26, 0.26, 0.26, 0.26, 0.26, 0.27, 0.27, 0.27, 0.27, 0.27, 0.18, 0.18, 0.18, 0.18, 0.18, 0.26, 0.26, 0.26, 0.26, 0.26, 0.14, 0.14, 0.14, 0.14, 0.14, 0.15, 0.15, 0.15, 0.15, 0.15, 0.29, 0.29, 0.29, 0.29, 0.29, 0.14, 0.14, 0.14, 0.14, 0.14, 0.11, 0.11, 0.11, 0.11, 0.11, 0.2, 0.2, 0.2, 0.2, 0.2, 0.32, 0.32, 0.32, 0.32, 0.32, 0.11, 0.11, 0.11, 0.11, 0.11, 0.08, 0.08, 0.08, 0.08, 0.08, 0.11, 0.11, 0.11, 0.11, 0.11, 0.15, 0.15, 0.15, 0.15, 0.15, 0.11, 0.11, 0.11, 0.11, 0.11, 0.13, 0.13, 0.13, 0.13, 0.13, 0.15, 0.15, 0.15, 0.15, 0.15, 0.16, 0.16, 0.16, 0.16, 0.16, 0.21, 0.21, 0.21, 0.21, 0.21, 0.23, 0.23, 0.23, 0.23, 0.23, 0.3, 0.3, 0.3, 0.3, 0.3, 0.35, 0.35, 0.35, 0.35, 0.35, 0.14, 0.14, 0.14, 0.14, 0.14, 0.19, 0.19, 0.19, 0.19, 0.19, 0.1, 0.1, 0.1, 0.1, 0.1, 0.16, 0.16, 0.16, 0.16, 0.16, 0.08, 0.08, 0.08, 0.08, 0.08, 0.15, 0.15, 0.15, 0.15, 0.15, 0.37, 0.37, 0.37, 0.37, 0.37, 0.45, 0.45, 0.45, 0.45, 0.45, 0.48, 0.48, 0.48, 0.48, 0.48, 0.48, 0.48, 0.48, 0.48, 0.48, 0.48, 0.48, 0.48, 0.48, 0.48, 0.53, 0.53, 0.53, 0.53, 0.53, 0.22, 0.22, 0.22, 0.22, 0.22, 0.18, 0.18, 0.18, 0.18, 0.18, 0.16, 0.16, 0.16]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 426 iterations"
    y-axis "llamacpp:requests_processing"
    x-axis "llamacpp:requests_processing" 1713187143 --> 1713187777
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 1.0, 1.0, 1.0, 1.0, 1.0, 6.0, 6.0, 6.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 1.0, 1.0, 1.0, 1.0, 1.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 2.0, 2.0, 2.0, 2.0, 2.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 1.0, 1.0, 1.0]
                    

Loading