ferrum_quantization/gguf/
source.rs

1use std::path::Path;
2use std::sync::Arc;
3
4use ferrum_interfaces::vnext::{
5    ElementType, RetainedHostMemoryRegion, StableHostMemory, VNextError, WeightComponentPayload,
6    WeightComponentRole, WeightComponentSource, WeightComponentSpec, WeightEncoding, WeightSchema,
7};
8use ferrum_types::{FerrumError, Result};
9
10use super::{GgmlDType, GgufFile, GgufHadamard, NativeGgufFile};
11use crate::safetensors_archive::transcode_dense_bytes;
12
13mod hadamard;
14use hadamard::HadamardRegistry;
15pub use hadamard::{hadamard_sign_component, hadamard_transform_spec};
16
17/// Schema-addressed, mmap-backed GGUF source for vNext static weights.
18/// Fixed-block payloads borrow the immutable file mapping without
19/// dequantization or repacking. Dense floating-point payloads are borrowed
20/// when their type matches the schema and materialized on a cold-path source
21/// request when the typed execution plan requires another floating-point type.
22pub struct GgufWeightComponentSource {
23    file: Arc<NativeGgufFile>,
24    source_file: String,
25    hadamard_registry: HadamardRegistry,
26}
27
28impl GgufWeightComponentSource {
29    pub fn open(path: impl AsRef<Path>) -> Result<Self> {
30        let source = Self::open_descriptor(path)?;
31        if source.file.hadamard().is_some() {
32            return Err(FerrumError::model(
33                super::hadamard::unsupported_execution().to_string(),
34            ));
35        }
36        Ok(source)
37    }
38
39    /// Admit source transforms only after exact binding to all corresponding
40    /// per-projection layout declarations. Ordinary `open` stays fail-closed.
41    pub fn open_with_schema(
42        path: impl AsRef<Path>,
43        schema: &WeightSchema,
44        expected_hadamard: Option<&GgufHadamard>,
45    ) -> Result<Self> {
46        let mut source = Self::open_descriptor(path)?;
47        if source.file.hadamard() != expected_hadamard {
48            return Err(FerrumError::model(
49                "GGUF transform metadata/sign values differ from the bound family identity",
50            ));
51        }
52        source.hadamard_registry = HadamardRegistry::bind(&source.file, schema)
53            .map_err(|error| FerrumError::model(error.to_string()))?;
54        Ok(source)
55    }
56
57    fn open_descriptor(path: impl AsRef<Path>) -> Result<Self> {
58        let path = path.as_ref();
59        let source_file = path
60            .file_name()
61            .and_then(|name| name.to_str())
62            .filter(|name| is_portable_source_file(name))
63            .ok_or_else(|| {
64                FerrumError::model(format!(
65                    "GGUF path must end in one portable UTF-8 file name: {}",
66                    path.display()
67                ))
68            })?
69            .to_owned();
70        let file = NativeGgufFile::open(path).map_err(|error| {
71            FerrumError::model(format!(
72                "open vNext GGUF source {}: {error}",
73                path.display()
74            ))
75        })?;
76        Ok(Self {
77            file: Arc::new(file),
78            source_file,
79            hadamard_registry: HadamardRegistry::default(),
80        })
81    }
82
83    pub fn file(&self) -> &NativeGgufFile {
84        &self.file
85    }
86
87    pub fn source_file(&self) -> &str {
88        &self.source_file
89    }
90}
91
92impl WeightComponentSource for GgufWeightComponentSource {
93    fn component<'source>(
94        &'source self,
95        component: &WeightComponentSpec,
96    ) -> std::result::Result<WeightComponentPayload<'source>, VNextError> {
97        if component.role == WeightComponentRole::TransformSigns
98            || self
99                .hadamard_registry
100                .components
101                .contains_key(&component.id)
102        {
103            let (expected, bytes) = self
104                .hadamard_registry
105                .components
106                .get(&component.id)
107                .ok_or_else(|| {
108                    invalid_component(component, "sign component is not in the source registry")
109                })?;
110            if expected != component {
111                return Err(invalid_component(
112                    component,
113                    "sign component differs from its registered source recipe",
114                ));
115            }
116            return WeightComponentPayload::new(
117                component,
118                expected.external_names[0].clone(),
119                self.source_file.clone(),
120                expected.dimensions.clone(),
121                ElementType::F32,
122                bytes.as_slice(),
123            );
124        }
125        let [external_name] = component.external_names.as_slice() else {
126            return Err(invalid_component(
127                component,
128                "GGUF components must bind exactly one physical tensor; combine tensors in PhysicalWeightLayout instead",
129            ));
130        };
131        let info = self.file.tensor_info(external_name).ok_or_else(|| {
132            invalid_component(
133                component,
134                format!("GGUF tensor {external_name:?} is absent"),
135            )
136        })?;
137        let bytes = self.file.tensor_byte_slice(external_name).ok_or_else(|| {
138            invalid_component(
139                component,
140                format!("GGUF tensor {external_name:?} has an invalid block or byte range"),
141            )
142        })?;
143
144        let (element_type, payload_bytes) = match &component.encoding {
145            WeightEncoding::Dense { element_type } => {
146                let WeightEncoding::Dense {
147                    element_type: actual,
148                } = &info.encoding
149                else {
150                    return Err(invalid_component(
151                        component,
152                        format!(
153                            "GGUF type {} is quantized but the schema declares dense bytes",
154                            info.ggml_type
155                        ),
156                    ));
157                };
158                let dimensions = &info.dimensions;
159                if dimensions != &component.dimensions {
160                    return Err(invalid_component(
161                        component,
162                        format!(
163                            "GGUF dense tensor dimensions differ: source_dtype={:?} dimensions={dimensions:?}",
164                            info.ggml_type,
165                        ),
166                    ));
167                }
168                let materialized =
169                    transcode_dense_bytes(bytes, *actual, *element_type, external_name, None)?;
170                (*element_type, materialized)
171            }
172            WeightEncoding::BlockQuantized(spec) => {
173                spec.validate()?;
174                let WeightEncoding::BlockQuantized(actual) = &info.encoding else {
175                    return Err(invalid_component(
176                        component,
177                        format!(
178                            "GGUF type {} is not a fixed-block quantization format",
179                            info.ggml_type
180                        ),
181                    ));
182                };
183                let logical_elements = info.elements;
184                let block_width = u64::from(spec.logical_values_per_block);
185                let mut physical_dimensions = info.dimensions.clone();
186                let innermost = physical_dimensions.last_mut().ok_or_else(|| {
187                    invalid_component(
188                        component,
189                        "GGUF quantized tensor must have at least one axis",
190                    )
191                })?;
192                if !innermost.is_multiple_of(block_width) {
193                    return Err(invalid_component(
194                        component,
195                        format!(
196                            "GGUF innermost dimension {innermost} is not divisible by block width {block_width}"
197                        ),
198                    ));
199                }
200                *innermost /= block_width;
201                if actual != spec
202                    || !logical_elements.is_multiple_of(block_width)
203                    || physical_dimensions != component.dimensions
204                {
205                    return Err(invalid_component(
206                        component,
207                        format!(
208                            "GGUF block ABI differs: file_type={} encoding={actual:?} logical_elements={logical_elements} physical_dimensions={physical_dimensions:?}",
209                            info.ggml_type,
210                        ),
211                    ));
212                }
213                (ElementType::U8, bytes.into())
214            }
215            WeightEncoding::DenseAffine { .. } => {
216                return Err(invalid_component(
217                    component,
218                    "GGUF source values are already transformed and cannot apply a dense affine source transform",
219                ));
220            }
221            WeightEncoding::Quantized(_) => {
222                return Err(invalid_component(
223                    component,
224                    "GGUF fixed-block bytes cannot satisfy a separate-component quantization encoding",
225                ));
226            }
227        };
228
229        let retained_host_memory =
230            if payload_bytes.as_ptr() == bytes.as_ptr() && payload_bytes.len() == bytes.len() {
231                let (offset_bytes, length_bytes) =
232                    self.file.tensor_byte_range(external_name).ok_or_else(|| {
233                        invalid_component(component, "GGUF tensor byte range is invalid")
234                    })?;
235                Some(RetainedHostMemoryRegion::new(
236                    Arc::clone(&self.file),
237                    offset_bytes,
238                    length_bytes,
239                )?)
240            } else {
241                None
242            };
243        let payload = WeightComponentPayload::new(
244            component,
245            external_name.clone(),
246            self.source_file.clone(),
247            component.dimensions.clone(),
248            element_type,
249            payload_bytes,
250        )?;
251        match retained_host_memory {
252            Some(retained) => payload.with_retained_host_memory(retained),
253            None => Ok(payload),
254        }
255    }
256}
257
258// SAFETY: GgufFile owns an immutable Mmap whose address and length do not
259// change during its lifetime.
260unsafe impl StableHostMemory for GgufFile {
261    fn stable_bytes(&self) -> &[u8] {
262        self.mmap_bytes()
263    }
264}
265
266// SAFETY: NativeGgufFile owns the same immutable mapping lifetime contract.
267unsafe impl StableHostMemory for NativeGgufFile {
268    fn stable_bytes(&self) -> &[u8] {
269        self.mmap_bytes()
270    }
271}
272
273pub fn block_quantization_format(dtype: GgmlDType) -> Option<&'static str> {
274    match dtype {
275        GgmlDType::Q4_0 => Some("quantization.gguf.q4-0"),
276        GgmlDType::Q4_1 => Some("quantization.gguf.q4-1"),
277        GgmlDType::Q5_0 => Some("quantization.gguf.q5-0"),
278        GgmlDType::Q5_1 => Some("quantization.gguf.q5-1"),
279        GgmlDType::Q8_0 => Some("quantization.gguf.q8-0"),
280        GgmlDType::Q8_1 => Some("quantization.gguf.q8-1"),
281        GgmlDType::Q2K => Some("quantization.gguf.q2-k"),
282        GgmlDType::Q3K => Some("quantization.gguf.q3-k"),
283        GgmlDType::Q4K => Some("quantization.gguf.q4-k"),
284        GgmlDType::Q5K => Some("quantization.gguf.q5-k"),
285        GgmlDType::Q6K => Some("quantization.gguf.q6-k"),
286        GgmlDType::Q8K => Some("quantization.gguf.q8-k"),
287        GgmlDType::F16 | GgmlDType::BF16 | GgmlDType::F32 => None,
288    }
289}
290
291fn is_portable_source_file(name: &str) -> bool {
292    !name.is_empty()
293        && !name.contains(['/', '\\'])
294        && !matches!(name, "." | "..")
295        && !name.bytes().any(|byte| byte.is_ascii_control())
296}
297
298fn invalid_component(component: &WeightComponentSpec, reason: impl Into<String>) -> VNextError {
299    VNextError::InvalidExecutionPlan {
300        reason: format!(
301            "GGUF component `{}` does not match its typed source: {}",
302            component.id,
303            reason.into()
304        ),
305    }
306}
ferrum_quantization/gguf/source.rs

ferrum_quantization/gguf/
source.rs
