ik-llama-cpp-sys 0.1.4

Low level bindings to ik_llama.cpp (ikawrakow's SOTA-quant fork)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

#include "iqk/iqk_config.h"

#if defined IQK_IMPLEMENT && defined GGML_IQK_FLASH_ATTENTION

#include "iqk/fa/iqk_fa_templates.h"

IQK_FA_CASE(iqk_fa_256_256) {

    auto type_k = ggml_type(int_type_k);
    auto type_v = ggml_type(int_type_v);

    stride_q /= sizeof(float); // q stride as float
    auto ck = (const char *)k;
    auto cv = (const char *)v;
    auto cm = (const char *)mask;

#ifdef __AVX512BF16__
    if (type_k == GGML_TYPE_BF16) {
        if (type_v != GGML_TYPE_BF16) return false; // we do not support mixing bf16 k-cache with other types
        if (nk%64 == 0) {
            iqk_flash_helper_T<256, 256, 64>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv,
                    q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S);
            return true;
        }
        iqk_flash_helper_T<256, 256, 32>(nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv,
                    q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S);
        return true;
    }
#endif

    if (nk%128 == 0) {
        return iqk_flash_helper_T<256, 256, 128>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv,
                q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S);
    }
    if (nk%64 == 0) {
        return iqk_flash_helper_T<256, 256, 64>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv,
                q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S);
    }

    return iqk_flash_helper_T<256, 256, 32>(type_k, type_v, nq, nk, stride_q, stride_k, stride_v, stride_m, stride_qkv,
                q, ck, cv, cm, scale, softcap, qkv, sinkf, M, S);

}

#endif