mistralrs-quant 0.8.1

Fast, flexible LLM inference.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

use half::f16;

#[allow(dead_code)]
extern "C" {
    pub(crate) fn reconstruct_exllama(
        b_q_weight: *const u32,
        b_qzeros: *const u32,
        b_scales: *const f16,
        b_q_perm: *const i32,
        out: *mut f16,
        size_k: i32,
        size_n: i32,
        groups: i32,
        bit: i32,
    );

    pub(crate) fn reconstruct_gptq(
        b_q_weight: *const u32,
        b_qzeros: *const u32,
        b_scales: *const f16,
        b_q_perm: *const i32,
        out: *mut f16,
        size_k: i32,
        size_n: i32,
        groups: i32,
        bit: i32,
    );

    pub(crate) fn gemm_half_q_half_cuda_part(
        a: *const f16,
        b_q_weight: *const u32,
        b_qzeros: *const u32,
        b_scales: *const f16,
        b_q_perm: *const i32,
        out: *mut f16,
        m: i32,
        n: i32,
        k: i32,
        m_count: i32,
        groups: i32,
        bit: i32,
    );

    pub(crate) fn gemm_half_q_half_alt(
        a: *const f16,
        b_q_weight: *const u32,
        b_qzeros: *const u32,
        b_scales: *const f16,
        b_g_idx: *const i32,
        out: *mut f16,
        m: i32,
        n: i32,
        k: i32,
        bit: i32,
    );
}