mistralrs-quant 0.8.1

Fast, flexible LLM inference.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

use std::os::raw::c_void;

pub(crate) const HAVE_MARLIN_KERNELS: bool = cfg!(has_marlin_kernels);

#[allow(dead_code)]
extern "C" {
    pub(crate) fn marlin_gptq_4bit_f16(
        inputs: *const c_void,
        weight: *const i32,
        scales: *const c_void,
        zeros: *const c_void,
        out: *const c_void,
        m: i32,
        k: i32,
        n: i32,
        workspace: *const c_void, //tensor with at least `n / 128 * max_par` entries that are all zero
        groupsize: i32,
        stream: i64,
    );

    pub(crate) fn marlin_gptq_4bit_bf16(
        inputs: *const c_void,
        weight: *const i32,
        scales: *const c_void,
        zeros: *const c_void,
        out: *const c_void,
        m: i32,
        k: i32,
        n: i32,
        workspace: *const c_void, //tensor with at least `n / 128 * max_par` entries that are all zero
        groupsize: i32,
        stream: i64,
    );

    pub(crate) fn marlin_awq_4bit_f16(
        inputs: *const c_void,
        weight: *const i32,
        scales: *const c_void,
        zeros: *const c_void,
        out: *const c_void,
        m: i32,
        k: i32,
        n: i32,
        workspace: *const c_void,
        groupsize: i32,
        stream: i64,
    );

    pub(crate) fn marlin_awq_4bit_bf16(
        inputs: *const c_void,
        weight: *const i32,
        scales: *const c_void,
        zeros: *const c_void,
        out: *const c_void,
        m: i32,
        k: i32,
        n: i32,
        workspace: *const c_void,
        groupsize: i32,
        stream: i64,
    );

    pub(crate) fn gptq_marlin_repack(
        weight: *const c_void,
        perm: *const c_void,
        result: *const c_void,
        k: i32,
        n: i32,
        bits: i32,
        stream: i64,
    );

    pub(crate) fn awq_marlin_repack(
        weight: *const c_void,
        perm: *const c_void,
        result: *const c_void,
        k: i32,
        n: i32,
        bits: i32,
        stream: i64,
    );
}