ferrum_quantization/
lib.rs

1//! Weight-format abstraction for Ferrum models.
2//!
3//! Separates "what is the weight matrix like" (dense f32, GPTQ int4, AWQ,
4//! GGUF, ...) from "what device does the math" (Backend) and "how does the
5//! model wire things together" (model code).
6//!
7//! Usage in model code:
8//! ```ignore
9//! let qkv: Box<dyn Linear<B>> = loader.load_linear("model.layers.0.self_attn.qkv_proj")?;
10//! qkv.forward(ctx, &input, &mut out, m);
11//! ```
12//!
13//! The `Linear` trait dispatches to the appropriate backend kernel
14//! (`B::gemm` for Dense, `B::gemm_gptq` for GPTQ, etc.) without the model
15//! having to branch on quantization type.
16
17#![forbid(unsafe_op_in_unsafe_fn)]
18
19pub mod block_fp8_safetensors_source;
20pub mod compressed_tensors_marlin_source;
21pub mod dense;
22pub mod gguf;
23pub mod gptq;
24pub mod gptq_marlin_source;
25pub mod loader;
26pub mod lora;
27pub mod mxfp4_safetensors_source;
28pub mod native_safetensors;
29pub mod quant_linear;
30pub mod safetensors_archive;
31pub mod traits;
32
33pub use block_fp8_safetensors_source::{
34    BlockFp8SafetensorsSource, BLOCK_FP8_E4M3_SOURCE_FORMAT_ID,
35};
36pub use compressed_tensors_marlin_source::{
37    CompressedTensorsMarlinSafetensorsSource, COMPRESSED_TENSORS_MARLIN_INT4_FORMAT_ID,
38    COMPRESSED_TENSORS_MARLIN_INT4_SYMMETRIC_FORMAT_ID,
39};
40pub use dense::DenseLinear;
41pub use gguf::{GgufFile, GgufLinear, GgufLoader, GgufWeightComponentSource};
42pub use gptq::{GptqLinear, StackedExpertLinear};
43pub use gptq_marlin_source::{GptqMarlinSafetensorsSource, GPTQ_MARLIN_INT4_FORMAT_ID};
44pub use loader::{PrefixedLoader, WeightLoader};
45pub use lora::LoraLinearRef;
46pub use mxfp4_safetensors_source::{Mxfp4SafetensorsSource, MXFP4_E2M1_E8M0_SOURCE_FORMAT_ID};
47pub use native_safetensors::NativeSafetensorsLoader;
48pub use quant_linear::QuantLinear;
49pub use safetensors_archive::{SafetensorsArchive, SafetensorsTensor};
50pub use traits::Linear;
51
52// Quant config types — populated from safetensors metadata or GGUF header.
53pub mod config;
54pub use config::{QuantConfig, QuantMethod};
ferrum_quantization/lib.rs

ferrum_quantization/
lib.rs
