Skip to main content

trustformers_models/weight_loading/
binding.rs

1//! Reusable binding helpers for `Model::load_pretrained` implementations.
2//!
3//! Every architecture in this crate maps HuggingFace tensor names onto its own
4//! layers, and every one of those maps needs the same three things: take a
5//! `[out, in]` weight (plus an optional `[out]` bias) for a `Linear`, take a
6//! `[dim]` vector for a norm, and check the shape rather than reshaping. Doing
7//! that by hand per model is how a loader ends up quietly skipping a parameter,
8//! so the primitives live here and each model contributes only its name map.
9//!
10//! The strictness of [`WeightBinder`](crate::weight_loading::checkpoint::WeightBinder) is
11//! preserved throughout: a tensor the checkpoint does not hold is *recorded as
12//! missing* and the parameter is left untouched, never filled with a
13//! substitute, and `WeightBinder::finish` turns the accumulated gaps into one
14//! error naming all of them.
15
16use trustformers_core::errors::{Result, TrustformersError};
17use trustformers_core::layers::{Embedding, LayerNorm, Linear};
18use trustformers_core::tensor::Tensor;
19
20use super::checkpoint::{Checkpoint, LoadReport, WeightBinder};
21
22/// Bind a `[out_features, in_features]` weight and an optional `[out_features]`
23/// bias into a [`Linear`].
24///
25/// Both HuggingFace's `nn.Linear` and this crate's [`Linear`] store the weight
26/// as `[out_features, in_features]`, so no transposition is involved; the shape
27/// is checked instead of assumed.
28///
29/// `expect_bias` distinguishes "this architecture has a bias here, report it if
30/// the checkpoint lacks it" from "this architecture has no bias, do not look".
31///
32/// # Errors
33///
34/// Fails when a tensor exists under the expected name but has the wrong shape.
35pub fn bind_linear(
36    binder: &mut WeightBinder<'_>,
37    name: &str,
38    out_features: usize,
39    in_features: usize,
40    expect_bias: bool,
41    layer: &mut Linear,
42) -> Result<()> {
43    if let Some(weight) =
44        binder.take_shaped(&format!("{name}.weight"), &[out_features, in_features])?
45    {
46        layer.set_weight(weight)?;
47    }
48    if expect_bias {
49        if let Some(bias) = binder.take_shaped(&format!("{name}.bias"), &[out_features])? {
50            layer.set_bias(bias)?;
51        }
52    }
53    Ok(())
54}
55
56/// Bind a `[num_embeddings, embedding_dim]` matrix into an [`Embedding`].
57///
58/// # Errors
59///
60/// Fails when the tensor exists but has the wrong shape.
61pub fn bind_embedding(
62    binder: &mut WeightBinder<'_>,
63    name: &str,
64    num_embeddings: usize,
65    embedding_dim: usize,
66    layer: &mut Embedding,
67) -> Result<()> {
68    if let Some(weight) =
69        binder.take_shaped(&format!("{name}.weight"), &[num_embeddings, embedding_dim])?
70    {
71        layer.set_weight(weight)?;
72    }
73    Ok(())
74}
75
76/// Take a `[dim]` normalisation weight, checking its shape.
77///
78/// Returned rather than assigned because each architecture wraps its norm in a
79/// different type; the caller stores it through that type's own setter.
80///
81/// # Errors
82///
83/// Fails when the tensor exists but has the wrong shape.
84pub fn take_norm_weight(
85    binder: &mut WeightBinder<'_>,
86    name: &str,
87    dim: usize,
88) -> Result<Option<Tensor>> {
89    binder.take_shaped(&format!("{name}.weight"), &[dim])
90}
91
92/// Take a `[dim]` normalisation bias, checking its shape.
93///
94/// # Errors
95///
96/// Fails when the tensor exists but has the wrong shape.
97pub fn take_norm_bias(
98    binder: &mut WeightBinder<'_>,
99    name: &str,
100    dim: usize,
101) -> Result<Option<Tensor>> {
102    binder.take_shaped(&format!("{name}.bias"), &[dim])
103}
104
105/// Bind a `[out, in]` task-head projection and its bias from a checkpoint.
106///
107/// Task heads are bound *after* the backbone, straight off the parsed
108/// [`Checkpoint`] rather than through a [`WeightBinder`], because the backbone's
109/// binder has already finished and deliberately tolerated the head's namespace.
110///
111/// A head the checkpoint does not carry is legitimate — a pretrained backbone
112/// ships without a fine-tuned head — but it is recorded in
113/// [`LoadReport::missing`] so the caller can see that the layer kept its
114/// constructor initialisation. A head that *is* present must reach the model:
115/// silently leaving it behind is exactly the failure this crate is being
116/// cleaned of.
117///
118/// # Errors
119///
120/// Fails when a tensor is present with the wrong shape.
121pub fn bind_head_linear(
122    checkpoint: &Checkpoint,
123    report: &mut LoadReport,
124    name: &str,
125    weight_shape: [usize; 2],
126    layer: &mut Linear,
127) -> Result<()> {
128    let weight_name = format!("{name}.weight");
129    match checkpoint.take_shaped(&weight_name, &weight_shape)? {
130        Some(weight) => {
131            layer.set_weight(weight)?;
132            report.mark_loaded(&weight_name);
133        },
134        None => report.note_absent(&weight_name),
135    }
136
137    let bias_name = format!("{name}.bias");
138    match checkpoint.take_shaped(&bias_name, &[weight_shape[0]])? {
139        Some(bias) => {
140            layer.set_bias(bias)?;
141            report.mark_loaded(&bias_name);
142        },
143        None => report.note_absent(&bias_name),
144    }
145    Ok(())
146}
147
148/// Bind a task-head [`LayerNorm`] from a checkpoint, recording what was found.
149///
150/// # Errors
151///
152/// Fails when a tensor is present with the wrong shape.
153pub fn bind_head_layer_norm(
154    checkpoint: &Checkpoint,
155    report: &mut LoadReport,
156    name: &str,
157    hidden_size: usize,
158    norm: &mut LayerNorm,
159) -> Result<()> {
160    let weight_name = format!("{name}.weight");
161    match checkpoint.take_shaped(&weight_name, &[hidden_size])? {
162        Some(weight) => {
163            norm.set_weight(weight)?;
164            report.mark_loaded(&weight_name);
165        },
166        None => report.note_absent(&weight_name),
167    }
168
169    let bias_name = format!("{name}.bias");
170    match checkpoint.take_shaped(&bias_name, &[hidden_size])? {
171        Some(bias) => {
172            norm.set_bias(bias)?;
173            report.mark_loaded(&bias_name);
174        },
175        None => report.note_absent(&bias_name),
176    }
177    Ok(())
178}
179
180/// The tensor shapes a decoder-only transformer's layers must have.
181///
182/// Grouped-query attention means `k_proj`/`v_proj` are narrower than
183/// `q_proj`: their output width is `num_key_value_heads * head_dim`. Getting
184/// that wrong is the single most common way a "successful" load ends up with
185/// transposed or truncated attention weights, so it is a first-class field here
186/// rather than an ad-hoc expression per model.
187#[derive(Debug, Clone, Copy, PartialEq, Eq)]
188pub struct DecoderShapes {
189    /// Vocabulary size — rows of the embedding matrix and the LM head.
190    pub vocab_size: usize,
191    /// Residual-stream width.
192    pub hidden_size: usize,
193    /// Feed-forward inner width.
194    pub intermediate_size: usize,
195    /// Number of decoder layers.
196    pub num_layers: usize,
197    /// Output width of `q_proj` (`num_attention_heads * head_dim`).
198    pub q_width: usize,
199    /// Output width of `k_proj`/`v_proj` (`num_key_value_heads * head_dim`).
200    pub kv_width: usize,
201}
202
203impl DecoderShapes {
204    /// Build the shapes for a model whose `head_dim` is `hidden_size / heads`.
205    pub fn new(
206        vocab_size: usize,
207        hidden_size: usize,
208        intermediate_size: usize,
209        num_layers: usize,
210        num_attention_heads: usize,
211        num_key_value_heads: usize,
212        head_dim: usize,
213    ) -> Self {
214        Self {
215            vocab_size,
216            hidden_size,
217            intermediate_size,
218            num_layers,
219            q_width: num_attention_heads * head_dim,
220            kv_width: num_key_value_heads * head_dim,
221        }
222    }
223}
224
225/// The checkpoint namespaces a task wrapper binds for itself.
226///
227/// # Why strictness has to be contextual
228///
229/// A bare encoder load has to tolerate whole head namespaces: HuggingFace ships
230/// `cls.predictions.*`, `classifier.*` and `qa_outputs.*` inside the same file
231/// as the encoder, so `BertModel::load_from_checkpoint` would fail on every real
232/// `bert-base-uncased` checkpoint if it refused them. That tolerance lives in
233/// each architecture's `ALLOWED_UNUSED_PREFIXES`.
234///
235/// A *task wrapper* is in the opposite position. It binds those very namespaces,
236/// so an entry left over inside one is not "a head this model does not have" —
237/// it is a name this model failed to recognise. Under the encoder's policy alone
238/// a typo such as `cls.predictions.transform.dens.weight` starts with `cls.`,
239/// lands in [`LoadReport::ignored`], and the load reports success while the
240/// dense layer silently keeps its random initialisation.
241///
242/// [`BoundNamespaces::verify`] closes exactly that gap: after the wrapper has
243/// bound its head, every checkpoint entry still unconsumed *inside a namespace
244/// the wrapper claims* becomes an error, while namespaces it does not claim stay
245/// as tolerant as before. A wrapper that claims `cls.predictions.` therefore
246/// still accepts a checkpoint's `cls.seq_relationship.*` NSP head, and a
247/// checkpoint with no head at all still loads (nothing is left over — the gap is
248/// reported through [`LoadReport::missing`] instead).
249#[derive(Debug, Clone, Copy)]
250pub struct BoundNamespaces<'a> {
251    /// Namespaces (matched with `starts_with`) this wrapper binds itself.
252    prefixes: &'a [&'a str],
253    /// Non-parameter buffer names (matched with `ends_with`) that stay tolerated
254    /// even inside a claimed namespace.
255    tolerated_suffixes: &'a [&'a str],
256}
257
258impl<'a> BoundNamespaces<'a> {
259    /// Claim a set of namespaces with no buffer exceptions.
260    pub const fn new(prefixes: &'a [&'a str]) -> Self {
261        Self {
262            prefixes,
263            tolerated_suffixes: &[],
264        }
265    }
266
267    /// Claim namespaces, excepting registered buffers matched by suffix.
268    ///
269    /// Needed where a head namespace legitimately carries a non-parameter entry
270    /// (a `position_ids` range, a cached mask) that no setter consumes.
271    pub const fn with_tolerated_suffixes(
272        prefixes: &'a [&'a str],
273        tolerated_suffixes: &'a [&'a str],
274    ) -> Self {
275        Self {
276            prefixes,
277            tolerated_suffixes,
278        }
279    }
280
281    /// The namespaces this wrapper claims.
282    pub fn prefixes(&self) -> &'a [&'a str] {
283        self.prefixes
284    }
285
286    /// Whether this entry falls inside a claimed namespace and is not an
287    /// explicitly tolerated buffer.
288    fn claims(&self, name: &str) -> bool {
289        self.prefixes.iter().any(|prefix| name.starts_with(prefix))
290            && !self.tolerated_suffixes.iter().any(|suffix| name.ends_with(suffix))
291    }
292
293    /// Fail when the checkpoint holds an unrecognised tensor inside a namespace
294    /// this wrapper binds.
295    ///
296    /// Call it *after* every head tensor has been bound, so that
297    /// [`LoadReport::mark_loaded`] has already moved the recognised names out of
298    /// [`LoadReport::ignored`].
299    ///
300    /// # Errors
301    ///
302    /// Fails when any entry under a claimed namespace was left unconsumed,
303    /// naming every offender and the namespaces that were claimed.
304    pub fn verify(&self, report: &LoadReport) -> Result<()> {
305        let unrecognised: Vec<&str> = report
306            .ignored
307            .iter()
308            .map(String::as_str)
309            .filter(|name| self.claims(name))
310            .collect();
311        if unrecognised.is_empty() {
312            return Ok(());
313        }
314        Err(TrustformersError::weight_load_error(format!(
315            "checkpoint holds {} tensor(s) this model does not recognise inside the head \
316             namespace(s) it binds ({}): {}. A bare encoder load tolerates a whole head \
317             namespace it does not bind, but this model binds these names, so an unconsumed \
318             entry here is a name it failed to match — most often a misspelling — not an \
319             absent head",
320            unrecognised.len(),
321            self.prefixes.join(", "),
322            unrecognised.join(", "),
323        )))
324    }
325}
326
327/// Checkpoint entries a decoder-only HuggingFace export carries that are not
328/// learnable parameters.
329///
330/// `rotary_emb.inv_freq` is a cached frequency table recomputed at construction
331/// time, `masked_bias`/`bias` under an attention block are causal-mask buffers,
332/// and `position_ids` is a range. Refusing a load over any of these would make
333/// every real checkpoint fail, while still refusing over an unknown *weight* is
334/// exactly the strictness worth keeping.
335pub const DECODER_BUFFER_SUFFIXES: &[&str] = &[
336    "rotary_emb.inv_freq",
337    "attn.masked_bias",
338    "attn.bias",
339    "embeddings.position_ids",
340    "rotary_emb.cos_cached",
341    "rotary_emb.sin_cached",
342];
343
344#[cfg(test)]
345mod tests {
346    use super::*;
347    use crate::weight_loading::checkpoint::{Checkpoint, UnusedTensors};
348    use crate::weight_loading::test_support::{build_safetensors, F32Tensor};
349
350    #[test]
351    fn bind_linear_copies_the_exact_checkpoint_values() {
352        let weight = F32Tensor::ramp("proj.weight", &[3, 2], 1.0);
353        let bias = F32Tensor::ramp("proj.bias", &[3], 40.0);
354        let bytes = build_safetensors(&[weight.clone(), bias.clone()]);
355        let checkpoint = Checkpoint::from_bytes(&bytes).expect("checkpoint must parse");
356
357        let mut layer = Linear::new(2, 3, true);
358        let mut binder = checkpoint.binder("");
359        bind_linear(&mut binder, "proj", 3, 2, true, &mut layer).expect("binding must succeed");
360        binder.finish(UnusedTensors::default()).expect("everything must be consumed");
361
362        assert_eq!(
363            layer.weight().data().expect("readable"),
364            weight.values,
365            "the bound weight must be the checkpoint's, byte for byte"
366        );
367        assert_eq!(
368            layer.bias().expect("bias must be set").data().expect("readable"),
369            bias.values
370        );
371    }
372
373    #[test]
374    fn bind_linear_reports_a_shape_mismatch_instead_of_reshaping() {
375        let bytes = build_safetensors(&[F32Tensor::ramp("proj.weight", &[6, 1], 1.0)]);
376        let checkpoint = Checkpoint::from_bytes(&bytes).expect("checkpoint must parse");
377        let mut layer = Linear::new(2, 3, false);
378        let mut binder = checkpoint.binder("");
379        let err = bind_linear(&mut binder, "proj", 3, 2, false, &mut layer)
380            .expect_err("a [6,1] tensor is not a [3,2] weight");
381        assert!(err.to_string().contains("[6, 1]"), "unexpected: {err}");
382    }
383
384    #[test]
385    fn a_missing_parameter_is_reported_rather_than_invented() {
386        let bytes = build_safetensors(&[F32Tensor::ramp("other.weight", &[1], 1.0)]);
387        let checkpoint = Checkpoint::from_bytes(&bytes).expect("checkpoint must parse");
388        let mut layer = Linear::new(2, 3, false);
389        let before = layer.weight().data().expect("readable");
390        let mut binder = checkpoint.binder("");
391        bind_linear(&mut binder, "proj", 3, 2, false, &mut layer).expect("binding must not fail");
392        assert_eq!(
393            layer.weight().data().expect("readable"),
394            before,
395            "an absent tensor must leave the layer untouched"
396        );
397        let err = binder
398            .finish(UnusedTensors::default())
399            .expect_err("the gap must be reported at finish time");
400        assert!(err.to_string().contains("proj.weight"), "unexpected: {err}");
401    }
402
403    #[test]
404    fn decoder_shapes_track_grouped_query_attention() {
405        // 8 query heads, 2 KV heads, head_dim 16: q is 128 wide, k/v are 32.
406        let shapes = DecoderShapes::new(100, 128, 256, 2, 8, 2, 16);
407        assert_eq!(shapes.q_width, 128);
408        assert_eq!(shapes.kv_width, 32);
409    }
410}