ferrum-kernels 0.7.0

Unified compute kernels (CUDA/Metal/CPU) and model runner for Ferrum inference
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

//! Ferrum unified compute kernels for high-performance inference.
//!
//! Provides the `Backend` trait and implementations for CUDA, Metal, and CPU.
//! On CUDA builds, kernels are compiled to PTX during `cargo build` and loaded
//! on demand at runtime.

pub mod backend;

pub mod linear;
pub use linear::Linear;

#[cfg(feature = "cuda")]
pub(crate) mod ptx {
    include!(concat!(env!("OUT_DIR"), "/ptx.rs"));
}

#[cfg(feature = "cuda")]
mod fused_add_rms_norm;
#[cfg(feature = "cuda")]
pub use fused_add_rms_norm::fused_add_rms_norm;

#[cfg(feature = "cuda")]
mod fused_silu_mul;
#[cfg(feature = "cuda")]
pub use fused_silu_mul::fused_silu_mul;

#[cfg(feature = "cuda")]
mod rms_norm;
#[cfg(feature = "cuda")]
pub use rms_norm::rms_norm;

#[cfg(feature = "cuda")]
mod rope;
#[cfg(feature = "cuda")]
pub use rope::rope;

#[cfg(feature = "cuda")]
mod decode_attention;
#[cfg(feature = "cuda")]
pub use decode_attention::decode_attention;

#[cfg(feature = "cuda")]
mod residual_add;
#[cfg(feature = "cuda")]
pub use residual_add::residual_add;

#[cfg(feature = "cuda")]
pub mod cublas;

#[cfg(feature = "cuda")]
pub mod decode_buffers;

#[cfg(feature = "cuda")]
pub mod weight_store;

#[cfg(feature = "cuda")]
pub mod cuda_graph;

#[cfg(feature = "cuda")]
pub mod quant;

#[cfg(feature = "cuda")]
pub mod marlin;

#[cfg(feature = "cuda")]
pub mod gpu_paged_kv;

#[cfg(feature = "cuda")]
pub mod cuda_decode;

#[cfg(feature = "cuda")]
pub mod nccl_comm;

#[cfg(feature = "cuda")]
pub mod tp_decode;