aprender-serve 0.67.0

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

// FALSIFY-QDOT-007/008 fixture, shared by the CPU (`quantize::tests::q5k_ggml`) and CUDA
// (`cuda::executor::tests_q5k_ggml`) Q5_K tests through include!().

/// 176 bytes: one ggml `block_q5_K` super-block, verbatim from a llama.cpp-quantized GGUF
/// (`Qwen3.5-0.8B-Q4_K_M.gguf`, tensor `blk.0.attn_qkv.weight`, super-block 1).
const GGML_Q5K_BLOCK: [u8; 176] = [
    0x2c, 0x03, 0x12, 0x14, 0xf3, 0xff, 0xf5, 0xf6, 0xb1, 0xbd, 0x7f, 0xa5, 0x45, 0x50, 0xef, 0x89,
    0x85, 0xbf, 0x07, 0x27, 0xef, 0xae, 0x4b, 0xaf, 0x87, 0x25, 0x96, 0x97, 0x36, 0x0e, 0x27, 0xd1,
    0x9f, 0x1f, 0x0e, 0x8f, 0x97, 0x05, 0x27, 0x06, 0x0d, 0x36, 0x9e, 0x3c, 0x4f, 0x25, 0x87, 0xaf,
    0xd5, 0x45, 0xf7, 0x38, 0x16, 0x0e, 0x08, 0x45, 0x75, 0xd6, 0x4a, 0x53, 0x4e, 0x10, 0x23, 0xfa,
    0x45, 0x67, 0x73, 0x46, 0x51, 0xe9, 0x34, 0x3f, 0x08, 0x2e, 0x4e, 0xcd, 0x6e, 0xc9, 0x2f, 0x54,
    0x00, 0xe7, 0x5b, 0x75, 0x9a, 0x4b, 0x30, 0xdc, 0xdf, 0xc1, 0xfd, 0xe1, 0x0a, 0x2a, 0xd1, 0x6d,
    0x2a, 0x62, 0x46, 0x8b, 0xbc, 0x83, 0xd6, 0xf6, 0xc7, 0xc8, 0x08, 0x09, 0x1b, 0x85, 0xa4, 0x88,
    0xe0, 0x10, 0x6a, 0x5e, 0x5f, 0x3e, 0x8f, 0x0c, 0xba, 0xed, 0x30, 0x53, 0x44, 0xdd, 0x8d, 0xf1,
    0xc2, 0xc1, 0x68, 0xe8, 0x7b, 0x09, 0x3a, 0x6e, 0xd9, 0x53, 0x58, 0x1f, 0x4d, 0x56, 0x3b, 0x2d,
    0x0d, 0xdc, 0xea, 0xdf, 0xef, 0x1f, 0x22, 0x47, 0x2c, 0x07, 0xe9, 0x2c, 0xeb, 0xca, 0x80, 0x02,
    0x1b, 0xb0, 0xe9, 0x79, 0x2e, 0xea, 0xc2, 0xfd, 0xfb, 0xb9, 0x2d, 0xcb, 0xe8, 0xe8, 0x98, 0x2b,
];
/// gguf-py `dequantize(block, Q5_K)` of `GGML_Q5K_BLOCK`: the 256 values in ggml order.
const GGML_Q5K_EXPECTED: [f32; 256] = [
    0.003_142_595_3, 0.003_142_595_3, 0.008_079_29, 0.010_547_638, 0.005_610_943, -0.014_135_838, 0.010_547_638, 0.003_142_595_3,
    0.003_142_595_3, 0.005_610_943, -0.024_009_228, -0.001_794_099_8, -0.014_135_838, -0.048_692_703, -0.001_794_099_8, 0.015_484_333,
    0.003_142_595_3, 0.008_079_29, -0.041_287_66, 0.005_610_943, -0.006_730_795, 0.013_015_985_5, 0.000_674_247_74, -0.011_667_49,
    0.010_547_638, -0.014_135_838, -0.014_135_838, -0.016_604_185, 0.025_357_723, 0.013_015_985_5, 0.027_826_07, 0.000_674_247_74,
    -0.020_978_69, 0.000_365_257_26, 0.033_905_745, -0.002_683_878, -0.008_782_148, -0.011_831_284, -0.011_831_284, 0.000_365_257_26,
    0.009_512_663, -0.020_978_69, 0.000_365_257_26, 0.003_414_392_5, 0.000_365_257_26, -0.008_782_148, -0.005_733_013, -0.014_880_419,
    0.000_365_257_26, 0.006_463_527_7, 0.009_512_663, 0.000_365_257_26, 0.003_414_392_5, -0.017_929_554, -0.002_683_878, -0.002_683_878,
    -0.060_617_447, -0.005_733_013, 0.000_365_257_26, -0.024_027_824, 0.006_463_527_7, -0.024_027_824, -0.005_733_013, 0.003_414_392_5,
    -0.021_562_576, -0.003_606_557_8, 0.006_654_024, -0.008_736_849, 0.004_088_878_6, 0.006_654_024, -0.062_604_904, 0.009_219_17,
    0.016_914_606, -0.018_997_43, 0.011_784_315, -0.018_997_43, 0.004_088_878_6, 0.004_088_878_6, -0.018_997_43, -0.029_258_013,
    0.004_088_878_6, -0.016_432_285, -0.006_171_703_3, 0.006_654_024, 0.009_219_17, -0.013_867_14, -0.006_171_703_3, -0.006_171_703_3,
    -0.003_606_557_8, -0.001_041_412_4, -0.001_041_412_4, 0.001_523_733_1, 0.006_654_024, -0.008_736_849, -0.011_301_994, -0.001_041_412_4,
    -0.036_767_96, 0.041_638_374, -0.023_700_237, -0.018_473_148, 0.028_570_652, 0.015_502_93, 0.012_889_385, 0.039_024_83,
    -0.002_791_881_6, -0.005_405_426, 0.002_435_207_4, -0.000_178_337_1, -0.036_767_96, 0.010_275_841, -0.002_791_881_6, -0.021_086_693,
    0.010_275_841, 0.020_730_019, 0.015_502_93, 0.025_957_108, -0.008_018_970_5, -0.015_859_604, -0.002_791_881_6, 0.002_435_207_4,
    0.036_411_285, -0.005_405_426, 0.005_048_752, 0.005_048_752, 0.007_662_296_3, -0.015_859_604, -0.010_632_515, 0.025_957_108,
    -0.035_774_23, 0.005_268_097, -0.010_122_776, 0.000_137_805_94, 0.002_702_951_4, 0.000_137_805_94, 0.002_702_951_4, -0.004_992_485,
    -0.010_122_776, -0.002_427_339_6, 0.005_268_097, 0.012_963_533, 0.015_528_679, -0.002_427_339_6, -0.002_427_339_6, 0.007_833_242,
    0.010_398_388, 0.007_833_242, -0.015_253_067, -0.015_253_067, 0.033_484_697, -0.012_687_921_5, -0.010_122_776, 0.000_137_805_94,
    -0.012_687_921_5, 0.012_963_533, 0.025_789_26, 0.043_745_28, -0.002_427_339_6, -0.020_383_358, -0.007_557_630_5, -0.002_427_339_6,
    -0.004_243_850_7, 0.002_725_601_2, -0.022_829_056, 0.012_018_204, 0.012_018_204, 0.007_371_902_5, -0.018_182_755, 0.000_402_450_56,
    -0.011_213_303, 0.032_926_56, -0.029_798_508, -0.025_152_206, 0.009_695_053, -0.006_567_001_3, 0.018_987_656, -0.001_920_700_1,
    -0.008_890_152, -0.008_890_152, -0.022_829_056, -0.004_243_850_7, -0.020_505_905, -0.036_767_96, 0.007_371_902_5, -0.022_829_056,
    -0.006_567_001_3, 0.012_018_204, -0.025_152_206, 0.002_725_601_2, -0.027_475_357, 0.012_018_204, -0.029_798_508, 0.005_048_752,
    0.009_826_899, 0.006_777_763_4, 0.000_679_492_95, 0.015_925_169, 0.064_711_33, 0.015_925_169, 0.025_072_575, -0.008_467_913,
    0.006_777_763_4, -0.008_467_913, -0.002_369_642_3, 0.006_777_763_4, 0.003_728_628_2, 0.000_679_492_95, -0.029_811_86, 0.025_072_575,
    0.003_728_628_2, -0.029_811_86, -0.002_369_642_3, -0.002_369_642_3, 0.012_876_034, 0.000_679_492_95, -0.023_713_589, 0.009_826_899,
    0.003_728_628_2, -0.002_369_642_3, 0.009_826_899, 0.003_728_628_2, 0.043_367_386, -0.005_418_777_5, -0.005_418_777_5, 0.003_728_628_2,
    0.004_390_716_6, 0.040_254_354, -0.001_126_766_2, -0.003_885_507_6, 0.043_013_096, 0.007_149_458, -0.034_231_663, 0.015_425_682,
    0.009_908_199, -0.039_749_146, 0.043_013_096, 0.009_908_199, -0.001_126_766_2, -0.006_644_249, -0.017_679_214, 0.004_390_716_6,
    0.007_149_458, -0.009_402_99, -0.001_126_766_2, 0.023_701_906, 0.009_908_199, -0.001_126_766_2, -0.006_644_249, 0.001_631_975_2,
    0.001_631_975_2, -0.009_402_99, 0.009_908_199, -0.006_644_249, -0.001_126_766_2, -0.001_126_766_2, 0.029_219_389, 0.009_908_199,
];