Skip to main content

trustformers_models/command_r/
model.rs

1use crate::command_r::config::CommandRConfig;
2use crate::common::ActivationType;
3use crate::generation_utils::GenerationUtils;
4use crate::weight_loading::{Checkpoint, LoadReport};
5use scirs2_core::random::{thread_rng, Rng}; // SciRS2 Integration Policy
6use trustformers_core::{
7    errors::{invalid_config, tensor_op_error, Result, TrustformersError},
8    layers::{Embedding, LayerNorm, Linear},
9    tensor::Tensor,
10    traits::{Config, Layer, Model},
11};
12
13/// Command R Rotary Position Embedding.
14///
15/// The rotation follows the HuggingFace `rotate_half` convention: for a head
16/// vector `x` of size `dim`, channel `i` is paired with channel `i + dim/2` and
17/// the pair is rotated by `position * inv_freq[i]`.
18#[derive(Debug, Clone)]
19pub struct CommandRRoPE {
20    dim: usize,
21    max_seq_len: usize,
22    base: f32,
23    /// `inv_freq[i] = base^(-2i/dim)` for `i in 0..dim/2`
24    inv_freq: Vec<f32>,
25}
26
27impl CommandRRoPE {
28    pub fn new(dim: usize, max_seq_len: usize, base: f32) -> Result<Self> {
29        if dim < 2 || !dim.is_multiple_of(2) {
30            return Err(invalid_config(
31                "CommandRRoPE::new",
32                format!("head dimension must be even and >= 2, got {dim}"),
33            ));
34        }
35        let inv_freq = (0..dim)
36            .step_by(2)
37            .map(|i| 1.0 / base.powf(i as f32 / dim as f32))
38            .collect::<Vec<f32>>();
39
40        Ok(Self {
41            dim,
42            max_seq_len,
43            base,
44            inv_freq,
45        })
46    }
47
48    /// Head dimension this RoPE was built for.
49    pub fn dim(&self) -> usize {
50        self.dim
51    }
52
53    /// Maximum position this RoPE accepts.
54    pub fn max_seq_len(&self) -> usize {
55        self.max_seq_len
56    }
57
58    /// RoPE base frequency (`theta`).
59    pub fn base(&self) -> f32 {
60        self.base
61    }
62
63    /// Inverse frequencies, one per rotated channel pair.
64    pub fn inv_freq(&self) -> &[f32] {
65        &self.inv_freq
66    }
67
68    /// Build the `cos`/`sin` tables for the given absolute positions.
69    ///
70    /// Both tensors have shape `[positions.len(), dim / 2]`.
71    pub fn cos_sin(&self, positions: &[usize]) -> Result<(Tensor, Tensor)> {
72        let half = self.dim / 2;
73        let mut cos_vals = Vec::with_capacity(positions.len() * half);
74        let mut sin_vals = Vec::with_capacity(positions.len() * half);
75        for &pos in positions {
76            for freq in &self.inv_freq {
77                let angle = pos as f32 * freq;
78                cos_vals.push(angle.cos());
79                sin_vals.push(angle.sin());
80            }
81        }
82        Ok((
83            Tensor::from_vec(cos_vals, &[positions.len(), half])?,
84            Tensor::from_vec(sin_vals, &[positions.len(), half])?,
85        ))
86    }
87
88    /// Rotate a flat `[batch, seq_len, num_heads, head_dim]` buffer in place.
89    ///
90    /// `positions[t]` is the **absolute** position of token `t`, so incremental
91    /// decoding with a KV cache rotates the new token with its true position.
92    pub fn rotate_in_place(
93        &self,
94        data: &mut [f32],
95        batch: usize,
96        num_heads: usize,
97        head_dim: usize,
98        positions: &[usize],
99    ) -> Result<()> {
100        if head_dim != self.dim {
101            return Err(tensor_op_error(
102                "CommandRRoPE::rotate_in_place",
103                format!("head_dim {head_dim} does not match RoPE dim {}", self.dim),
104            ));
105        }
106        let seq_len = positions.len();
107        let expected = batch * seq_len * num_heads * head_dim;
108        if data.len() != expected {
109            return Err(tensor_op_error(
110                "CommandRRoPE::rotate_in_place",
111                format!("expected {expected} elements, got {}", data.len()),
112            ));
113        }
114        if let Some(&max_pos) = positions.iter().max() {
115            if max_pos >= self.max_seq_len {
116                return Err(tensor_op_error(
117                    "CommandRRoPE::rotate_in_place",
118                    format!(
119                        "position {max_pos} exceeds max_sequence_length {}",
120                        self.max_seq_len
121                    ),
122                ));
123            }
124        }
125
126        let half = head_dim / 2;
127        for b in 0..batch {
128            for (t, &pos) in positions.iter().enumerate() {
129                for head in 0..num_heads {
130                    let base = ((b * seq_len + t) * num_heads + head) * head_dim;
131                    for i in 0..half {
132                        let angle = pos as f32 * self.inv_freq[i];
133                        let (sin_val, cos_val) = angle.sin_cos();
134                        let x0 = data[base + i];
135                        let x1 = data[base + i + half];
136                        data[base + i] = x0 * cos_val - x1 * sin_val;
137                        data[base + i + half] = x0 * sin_val + x1 * cos_val;
138                    }
139                }
140            }
141        }
142        Ok(())
143    }
144}
145
146/// Command R Attention layer
147#[derive(Debug, Clone)]
148pub struct CommandRAttention {
149    #[allow(dead_code)]
150    config: CommandRConfig,
151    hidden_size: usize,
152    num_heads: usize,
153    num_key_value_heads: usize,
154    head_dim: usize,
155
156    q_proj: Linear,
157    k_proj: Linear,
158    v_proj: Linear,
159    o_proj: Linear,
160
161    rope: CommandRRoPE,
162    attention_dropout: f32,
163    #[allow(dead_code)]
164    use_flash_attention: bool,
165}
166
167impl CommandRAttention {
168    pub fn new(config: &CommandRConfig) -> Result<Self> {
169        let hidden_size = config.hidden_size;
170        let num_heads = config.num_attention_heads;
171        let num_key_value_heads = config.num_key_value_heads;
172        let head_dim = config.head_dim();
173
174        let q_proj = Linear::new(hidden_size, num_heads * head_dim, config.use_bias);
175        let k_proj = Linear::new(hidden_size, num_key_value_heads * head_dim, config.use_bias);
176        let v_proj = Linear::new(hidden_size, num_key_value_heads * head_dim, config.use_bias);
177        let o_proj = Linear::new(num_heads * head_dim, hidden_size, config.use_bias);
178
179        let rope = CommandRRoPE::new(head_dim, config.max_sequence_length, config.rope_theta)?;
180
181        Ok(Self {
182            config: config.clone(),
183            hidden_size,
184            num_heads,
185            num_key_value_heads,
186            head_dim,
187            q_proj,
188            k_proj,
189            v_proj,
190            o_proj,
191            rope,
192            attention_dropout: config.attention_dropout,
193            use_flash_attention: config.use_flash_attention,
194        })
195    }
196
197    /// Causal grouped-query attention with a real KV cache.
198    ///
199    /// * `hidden_states` — `[batch, seq_len, hidden_size]`
200    /// * `attention_mask` — optional **additive** mask broadcastable to
201    ///   `[batch, num_heads, seq_len, total_len]`
202    /// * `position_ids` — absolute positions of the tokens in `hidden_states`;
203    ///   when it does not carry one entry per token the positions default to
204    ///   `past_len .. past_len + seq_len`
205    /// * `past_key_value` — cached keys/values shaped
206    ///   `[batch, past_len, num_key_value_heads, head_dim]`
207    ///
208    /// The returned cache is the **concatenation** of the past and the freshly
209    /// computed keys/values, so incremental decoding attends to every token seen
210    /// so far.
211    pub fn forward(
212        &self,
213        hidden_states: &Tensor,
214        attention_mask: Option<&Tensor>,
215        position_ids: &Tensor,
216        past_key_value: Option<(&Tensor, &Tensor)>,
217    ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
218        let shape = hidden_states.shape().to_vec();
219        if shape.len() != 3 {
220            return Err(tensor_op_error(
221                "CommandRAttention::forward",
222                format!("expected [batch, seq_len, hidden_size], got {shape:?}"),
223            ));
224        }
225        let (batch_size, seq_len, hidden) = (shape[0], shape[1], shape[2]);
226        if hidden != self.hidden_size {
227            return Err(tensor_op_error(
228                "CommandRAttention::forward",
229                format!(
230                    "input hidden size {hidden} does not match config hidden size {}",
231                    self.hidden_size
232                ),
233            ));
234        }
235
236        // Length of the cached prefix (0 when decoding from scratch).
237        let past_len = match past_key_value {
238            Some((past_key, past_value)) => {
239                let key_shape = past_key.shape().to_vec();
240                let value_shape = past_value.shape().to_vec();
241                let expected_trailing = [batch_size, self.num_key_value_heads, self.head_dim];
242                let valid = |s: &[usize]| {
243                    s.len() == 4
244                        && s[0] == expected_trailing[0]
245                        && s[2] == expected_trailing[1]
246                        && s[3] == expected_trailing[2]
247                };
248                if !valid(&key_shape) || !valid(&value_shape) || key_shape[1] != value_shape[1] {
249                    return Err(tensor_op_error(
250                        "CommandRAttention::forward",
251                        format!(
252                            "past key/value must be [{}, past_len, {}, {}], got {key_shape:?} / {value_shape:?}",
253                            batch_size, self.num_key_value_heads, self.head_dim
254                        ),
255                    ));
256                }
257                key_shape[1]
258            },
259            None => 0,
260        };
261
262        // Absolute positions for the incoming tokens.
263        let positions = Self::resolve_positions(position_ids, seq_len, past_len);
264
265        // Project to queries, keys, and values.
266        let query_states = self.q_proj.forward(hidden_states.clone())?;
267        let key_states = self.k_proj.forward(hidden_states.clone())?;
268        let value_states = self.v_proj.forward(hidden_states.clone())?;
269
270        // Apply RoPE to the queries and the *new* keys only (cached keys were
271        // already rotated when they were produced).
272        let mut query_data = query_states.data()?;
273        let mut key_data = key_states.data()?;
274        self.rope.rotate_in_place(
275            &mut query_data,
276            batch_size,
277            self.num_heads,
278            self.head_dim,
279            &positions,
280        )?;
281        self.rope.rotate_in_place(
282            &mut key_data,
283            batch_size,
284            self.num_key_value_heads,
285            self.head_dim,
286            &positions,
287        )?;
288
289        let new_key = Tensor::from_vec(
290            key_data,
291            &[batch_size, seq_len, self.num_key_value_heads, self.head_dim],
292        )?;
293        let new_value = value_states.reshape(&[
294            batch_size,
295            seq_len,
296            self.num_key_value_heads,
297            self.head_dim,
298        ])?;
299
300        // Append the new keys/values to the cache along the sequence axis.
301        let (key_cache, value_cache) = match past_key_value {
302            Some((past_key, past_value)) => (
303                Tensor::concat(&[past_key.clone(), new_key], 1)?,
304                Tensor::concat(&[past_value.clone(), new_value], 1)?,
305            ),
306            None => (new_key, new_value),
307        };
308
309        let total_len = past_len + seq_len;
310        let attn_output = self.scaled_dot_product_attention(
311            &query_data,
312            &key_cache,
313            &value_cache,
314            batch_size,
315            seq_len,
316            total_len,
317            past_len,
318            attention_mask,
319        )?;
320
321        let attn_output = attn_output.reshape(&[batch_size, seq_len, self.hidden_size])?;
322        let attn_output = self.o_proj.forward(attn_output)?;
323
324        Ok((attn_output, Some((key_cache, value_cache))))
325    }
326
327    /// Absolute positions for the incoming tokens.
328    ///
329    /// `position_ids` wins when it supplies one entry per token; otherwise the
330    /// positions continue the cached prefix.
331    fn resolve_positions(position_ids: &Tensor, seq_len: usize, past_len: usize) -> Vec<usize> {
332        if let Ok(values) = position_ids.data() {
333            if values.len() == seq_len {
334                return values.iter().map(|&p| p.max(0.0) as usize).collect();
335            }
336            if values.len() > seq_len && values.len() % seq_len == 0 {
337                // [batch, seq_len]: every row carries the same positions.
338                return values[..seq_len].iter().map(|&p| p.max(0.0) as usize).collect();
339            }
340        }
341        (past_len..past_len + seq_len).collect()
342    }
343
344    /// `softmax(mask(Q Kᵀ) / sqrt(head_dim)) V` over the cached keys/values.
345    ///
346    /// Queries are the flat `[batch, seq_len, num_heads, head_dim]` buffer that
347    /// has already been rotated; keys/values are the full cache
348    /// `[batch, total_len, num_key_value_heads, head_dim]`. Query head `h` reads
349    /// KV head `h / num_query_groups`, which is grouped-query attention.
350    ///
351    /// Attention dropout is deliberately not applied: this is an inference path
352    /// and randomised logits would make generation non-reproducible.
353    #[allow(clippy::too_many_arguments)]
354    fn scaled_dot_product_attention(
355        &self,
356        query_data: &[f32],
357        key_cache: &Tensor,
358        value_cache: &Tensor,
359        batch_size: usize,
360        seq_len: usize,
361        total_len: usize,
362        past_len: usize,
363        attention_mask: Option<&Tensor>,
364    ) -> Result<Tensor> {
365        let head_dim = self.head_dim;
366        let num_heads = self.num_heads;
367        let num_kv_heads = self.num_key_value_heads;
368        if num_kv_heads == 0 || !num_heads.is_multiple_of(num_kv_heads) {
369            return Err(invalid_config(
370                "CommandRAttention",
371                format!(
372                    "num_attention_heads {num_heads} must be a multiple of num_key_value_heads {num_kv_heads}"
373                ),
374            ));
375        }
376        let groups = num_heads / num_kv_heads;
377
378        let key_data = key_cache.data()?;
379        let value_data = value_cache.data()?;
380
381        // Optional additive mask, indexed as [batch?, head?, query, key].
382        let mask_data = match attention_mask {
383            Some(mask) => Some(mask.data()?),
384            None => None,
385        };
386        let mask_len = mask_data.as_ref().map(|m| m.len()).unwrap_or(0);
387
388        let scale = 1.0 / (head_dim as f32).sqrt();
389        let mut output = vec![0.0f32; batch_size * seq_len * num_heads * head_dim];
390        let mut scores = vec![0.0f32; total_len];
391
392        for b in 0..batch_size {
393            for head in 0..num_heads {
394                let kv_head = head / groups;
395                for q_idx in 0..seq_len {
396                    let q_base = ((b * seq_len + q_idx) * num_heads + head) * head_dim;
397                    // Causal limit: a query at cache slot past_len + q_idx may
398                    // read every key up to and including its own slot.
399                    let visible = past_len + q_idx + 1;
400                    let mut max_score = f32::NEG_INFINITY;
401                    for (key_idx, score) in scores.iter_mut().take(visible).enumerate() {
402                        let k_base =
403                            ((b * total_len + key_idx) * num_kv_heads + kv_head) * head_dim;
404                        let mut dot = 0.0f32;
405                        for d in 0..head_dim {
406                            dot += query_data[q_base + d] * key_data[k_base + d];
407                        }
408                        dot *= scale;
409                        if let Some(mask) = &mask_data {
410                            dot += Self::mask_value(
411                                mask, mask_len, b, head, q_idx, key_idx, batch_size, num_heads,
412                                seq_len, total_len,
413                            );
414                        }
415                        *score = dot;
416                        if dot > max_score {
417                            max_score = dot;
418                        }
419                    }
420
421                    let mut sum = 0.0f32;
422                    for score in scores.iter_mut().take(visible) {
423                        *score = (*score - max_score).exp();
424                        sum += *score;
425                    }
426                    let inv_sum = if sum > 0.0 { 1.0 / sum } else { 0.0 };
427
428                    let out_base = ((b * seq_len + q_idx) * num_heads + head) * head_dim;
429                    for (key_idx, score) in scores.iter().take(visible).enumerate() {
430                        let weight = score * inv_sum;
431                        if weight == 0.0 {
432                            continue;
433                        }
434                        let v_base =
435                            ((b * total_len + key_idx) * num_kv_heads + kv_head) * head_dim;
436                        for d in 0..head_dim {
437                            output[out_base + d] += weight * value_data[v_base + d];
438                        }
439                    }
440                }
441            }
442        }
443
444        Tensor::from_vec(output, &[batch_size, seq_len, num_heads, head_dim])
445    }
446
447    /// Read an additive mask entry, tolerating the common broadcast layouts
448    /// (`[q, k]`, `[batch, q, k]`, `[batch, 1, q, k]`, `[batch, heads, q, k]`).
449    #[allow(clippy::too_many_arguments)]
450    fn mask_value(
451        mask: &[f32],
452        mask_len: usize,
453        batch: usize,
454        head: usize,
455        q_idx: usize,
456        key_idx: usize,
457        batch_size: usize,
458        num_heads: usize,
459        seq_len: usize,
460        total_len: usize,
461    ) -> f32 {
462        let plane = seq_len * total_len;
463        if mask_len == plane {
464            return mask[q_idx * total_len + key_idx];
465        }
466        if mask_len == batch_size * plane {
467            return mask[batch * plane + q_idx * total_len + key_idx];
468        }
469        if mask_len == batch_size * num_heads * plane {
470            return mask[((batch * num_heads + head) * seq_len + q_idx) * total_len + key_idx];
471        }
472        0.0
473    }
474
475    /// Configured attention-dropout probability.
476    ///
477    /// Dropout is a training-time regulariser; this inference path never applies
478    /// it so that decoding stays reproducible.
479    pub fn attention_dropout(&self) -> f32 {
480        self.attention_dropout
481    }
482
483    pub fn parameter_count(&self) -> usize {
484        self.q_proj.parameter_count()
485            + self.k_proj.parameter_count()
486            + self.v_proj.parameter_count()
487            + self.o_proj.parameter_count()
488    }
489}
490
491/// Command R MLP (Feed-Forward Network)
492#[derive(Debug, Clone)]
493pub struct CommandRMLP {
494    #[allow(dead_code)]
495    config: CommandRConfig,
496    #[allow(dead_code)]
497    hidden_size: usize,
498    #[allow(dead_code)]
499    intermediate_size: usize,
500
501    gate_proj: Linear,
502    up_proj: Linear,
503    down_proj: Linear,
504
505    activation: ActivationType,
506}
507
508impl CommandRMLP {
509    pub fn new(config: &CommandRConfig) -> Result<Self> {
510        let hidden_size = config.hidden_size;
511        let intermediate_size = config.intermediate_size;
512
513        let gate_proj = Linear::new(hidden_size, intermediate_size, config.use_bias);
514        let up_proj = Linear::new(hidden_size, intermediate_size, config.use_bias);
515        let down_proj = Linear::new(intermediate_size, hidden_size, config.use_bias);
516
517        Ok(Self {
518            config: config.clone(),
519            hidden_size,
520            intermediate_size,
521            gate_proj,
522            up_proj,
523            down_proj,
524            // Parse once at construction; unknown identifiers fall back to GELU,
525            // matching the previous `_ => gate_output.gelu()` default arm.
526            activation: ActivationType::from_config_str_or(
527                &config.activation_function,
528                ActivationType::Gelu,
529            ),
530        })
531    }
532
533    pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
534        // Gate projection with activation
535        let gate_output = self.gate_proj.forward(x.clone())?;
536        let gate_output = self.activation.apply(&gate_output)?;
537
538        // Up projection
539        let up_output = self.up_proj.forward(x.clone())?;
540
541        // Element-wise multiplication
542        let intermediate = gate_output.mul(&up_output)?;
543
544        // Down projection
545        let output = self.down_proj.forward(intermediate)?;
546
547        Ok(output)
548    }
549
550    pub fn parameter_count(&self) -> usize {
551        self.gate_proj.parameter_count()
552            + self.up_proj.parameter_count()
553            + self.down_proj.parameter_count()
554    }
555}
556
557/// Command R Decoder Layer
558#[derive(Debug, Clone)]
559pub struct CommandRDecoderLayer {
560    #[allow(dead_code)]
561    config: CommandRConfig,
562    #[allow(dead_code)]
563    hidden_size: usize,
564
565    self_attn: CommandRAttention,
566    mlp: CommandRMLP,
567    input_layernorm: LayerNorm,
568    post_attention_layernorm: LayerNorm,
569}
570
571impl CommandRDecoderLayer {
572    pub fn new(config: &CommandRConfig) -> Result<Self> {
573        let hidden_size = config.hidden_size;
574
575        let self_attn = CommandRAttention::new(config)?;
576        let mlp = CommandRMLP::new(config)?;
577
578        let input_layernorm = LayerNorm::new(vec![hidden_size], config.rms_norm_eps)?;
579        let post_attention_layernorm = LayerNorm::new(vec![hidden_size], config.rms_norm_eps)?;
580
581        Ok(Self {
582            config: config.clone(),
583            hidden_size,
584            self_attn,
585            mlp,
586            input_layernorm,
587            post_attention_layernorm,
588        })
589    }
590
591    pub fn forward(
592        &self,
593        hidden_states: &Tensor,
594        attention_mask: Option<&Tensor>,
595        position_ids: &Tensor,
596        past_key_value: Option<(&Tensor, &Tensor)>,
597    ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
598        let residual = hidden_states.clone();
599
600        // Pre-attention layer norm
601        let hidden_states = self.input_layernorm.forward(hidden_states.clone())?;
602
603        // Self-attention
604        let (attn_output, present_key_value) =
605            self.self_attn
606                .forward(&hidden_states, attention_mask, position_ids, past_key_value)?;
607
608        // Add residual connection
609        let hidden_states = residual.add(&attn_output)?;
610        let residual = hidden_states.clone();
611
612        // Post-attention layer norm
613        let hidden_states = self.post_attention_layernorm.forward(hidden_states)?;
614
615        // MLP
616        let mlp_output = self.mlp.forward(&hidden_states)?;
617
618        // Add residual connection
619        let hidden_states = residual.add(&mlp_output)?;
620
621        Ok((hidden_states, present_key_value))
622    }
623
624    pub fn parameter_count(&self) -> usize {
625        self.self_attn.parameter_count()
626            + self.mlp.parameter_count()
627            + self.input_layernorm.parameter_count()
628            + self.post_attention_layernorm.parameter_count()
629    }
630}
631
632/// Command R Model
633#[derive(Debug, Clone)]
634pub struct CommandRModel {
635    config: CommandRConfig,
636    #[allow(dead_code)]
637    vocab_size: usize,
638    #[allow(dead_code)]
639    hidden_size: usize,
640    #[allow(dead_code)]
641    num_hidden_layers: usize,
642
643    embed_tokens: Embedding,
644    layers: Vec<CommandRDecoderLayer>,
645    norm: LayerNorm,
646
647    #[allow(dead_code)]
648    pad_token_id: Option<usize>,
649    #[allow(dead_code)]
650    bos_token_id: Option<usize>,
651    #[allow(dead_code)]
652    eos_token_id: Option<usize>,
653}
654
655impl CommandRModel {
656    pub fn new(config: &CommandRConfig) -> Result<Self> {
657        config.validate().map_err(|e| invalid_config("config_validation", &e))?;
658
659        let vocab_size = config.vocab_size;
660        let hidden_size = config.hidden_size;
661        let num_hidden_layers = config.num_hidden_layers;
662
663        let embed_tokens = Embedding::new(vocab_size, hidden_size, None)?;
664
665        let mut layers = Vec::new();
666        for _ in 0..num_hidden_layers {
667            layers.push(CommandRDecoderLayer::new(config)?);
668        }
669
670        let norm = LayerNorm::new(vec![hidden_size], config.rms_norm_eps)?;
671
672        Ok(Self {
673            config: config.clone(),
674            vocab_size,
675            hidden_size,
676            num_hidden_layers,
677            embed_tokens,
678            layers,
679            norm,
680            pad_token_id: config.pad_token_id,
681            bos_token_id: config.bos_token_id,
682            eos_token_id: config.eos_token_id,
683        })
684    }
685
686    pub fn forward(
687        &self,
688        input_ids: &Tensor,
689        attention_mask: Option<&Tensor>,
690        position_ids: Option<&Tensor>,
691        past_key_values: Option<&[(Tensor, Tensor)]>,
692    ) -> Result<CommandRModelOutput> {
693        let input_shape = input_ids.shape().to_vec();
694        if input_shape.len() != 2 {
695            return Err(tensor_op_error(
696                "CommandRModel::forward",
697                format!("expected input ids of shape [batch, seq_len], got {input_shape:?}"),
698            ));
699        }
700        let batch_size = input_shape[0];
701        let seq_len = input_shape[1];
702
703        // Length of the cached prefix, so positions continue where it left off.
704        let past_len = past_key_values
705            .and_then(|pkv| pkv.first())
706            .map(|(key, _)| {
707                let shape = key.shape().to_vec();
708                if shape.len() == 4 {
709                    shape[1]
710                } else {
711                    0
712                }
713            })
714            .unwrap_or(0);
715
716        // Create position IDs if not provided
717        let position_ids = if let Some(pos_ids) = position_ids {
718            pos_ids.clone()
719        } else {
720            let pos_ids: Vec<f32> =
721                (past_len..past_len + seq_len).map(|position| position as f32).collect();
722            Tensor::from_vec(pos_ids, &[1, seq_len])?
723        };
724
725        // Token embeddings
726        // Convert tensor to vector of token IDs
727        let input_ids_vec = match input_ids {
728            Tensor::I64(arr) => arr.iter().map(|&x| x as u32).collect::<Vec<u32>>(),
729            _ => {
730                return Err(tensor_op_error(
731                    "CommandRModel::forward",
732                    "Input IDs must be integer tensor",
733                ))
734            },
735        };
736        // Embedding lookup returns [batch * seq_len, hidden]; restore the batch axis.
737        let embedded = self.embed_tokens.forward(input_ids_vec)?;
738        let mut hidden_states =
739            embedded.reshape(&[batch_size, seq_len, self.config.hidden_size])?;
740
741        // Process through transformer layers
742        let mut present_key_values = Vec::new();
743        for (layer_idx, layer) in self.layers.iter().enumerate() {
744            let past_key_value = past_key_values.map(|pkv| (&pkv[layer_idx].0, &pkv[layer_idx].1));
745
746            let (layer_output, present_key_value) = layer.forward(
747                &hidden_states,
748                attention_mask,
749                &position_ids,
750                past_key_value,
751            )?;
752
753            hidden_states = layer_output;
754            if let Some(pkv) = present_key_value {
755                present_key_values.push(pkv);
756            }
757        }
758
759        // Final layer norm
760        let hidden_states = self.norm.forward(hidden_states)?;
761
762        Ok(CommandRModelOutput {
763            last_hidden_state: hidden_states,
764            past_key_values: if present_key_values.is_empty() {
765                None
766            } else {
767                Some(present_key_values)
768            },
769            hidden_states: None,
770            attentions: None,
771        })
772    }
773}
774
775impl Model for CommandRModel {
776    type Config = CommandRConfig;
777    type Input = Tensor;
778    type Output = Tensor;
779
780    fn forward(&self, input: Self::Input) -> Result<Self::Output> {
781        // Process input through model layers
782        let mut hidden_states = input;
783
784        // Pass through all decoder layers - note: layer.forward returns (hidden_states, past_key_value)
785        // For the Model trait implementation, we ignore past_key_values and use default params
786        let seq_len = hidden_states.shape().get(1).copied().unwrap_or(0);
787        let position_ids = Tensor::from_vec(
788            (0..seq_len).map(|position| position as f32).collect(),
789            &[1, seq_len],
790        )?;
791        for layer in &self.layers {
792            let (new_hidden_states, _) = layer.forward(
793                &hidden_states,
794                None, // attention_mask
795                &position_ids,
796                None, // past_key_value
797            )?;
798            hidden_states = new_hidden_states;
799        }
800
801        // Apply final normalization
802        hidden_states = self.norm.forward(hidden_states)?;
803
804        Ok(hidden_states)
805    }
806
807    /// Load a HuggingFace Command-R checkpoint (safetensors or `torch.save`).
808    ///
809    /// Every tensor is bound to a named parameter; nothing is invented. Tensors
810    /// the architecture does not know about make the load fail rather than being
811    /// logged and ignored, and parameters the checkpoint does not carry are
812    /// reported in the [`LoadReport`] instead of silently keeping their
813    /// randomly-initialised values.
814    fn load_pretrained(&mut self, reader: &mut dyn std::io::Read) -> Result<()> {
815        let checkpoint = Checkpoint::from_reader(reader)?;
816        self.load_checkpoint(&checkpoint, &["lm_head."])?;
817        Ok(())
818    }
819
820    fn get_config(&self) -> &Self::Config {
821        &self.config
822    }
823
824    fn num_parameters(&self) -> usize {
825        let embed_params = self.embed_tokens.parameter_count();
826        let layers_params: usize = self.layers.iter().map(|layer| layer.parameter_count()).sum();
827        let norm_params = self.norm.parameter_count();
828
829        embed_params + layers_params + norm_params
830    }
831}
832
833impl CommandRModel {
834    /// Bind a parsed checkpoint into this model.
835    ///
836    /// Accepts both `model.*`-prefixed (`CohereForCausalLM`) and bare
837    /// (`CohereModel`) HuggingFace layouts. `allowed_unused_prefixes` names the
838    /// checkpoint namespaces this base model legitimately ignores.
839    ///
840    /// Command-R shares one layer norm across the parallel attention/MLP block.
841    /// This implementation runs attention and MLP sequentially with two norms, so
842    /// a released Cohere checkpoint carries no `post_attention_layernorm`; that
843    /// norm keeps its unit scale and the block therefore does **not** reproduce
844    /// Cohere's parallel-residual arithmetic bit for bit. Every tensor the
845    /// checkpoint does carry is bound exactly, and nothing is invented.
846    pub fn load_checkpoint(
847        &mut self,
848        checkpoint: &Checkpoint,
849        allowed_unused_prefixes: &[&str],
850    ) -> Result<LoadReport> {
851        let prefix = checkpoint.detect_prefix(&["model.", ""], "embed_tokens.weight")?;
852        let mut binder = checkpoint.binder(&prefix);
853
854        let hidden = self.config.hidden_size;
855        let head_dim = self.config.head_dim();
856        let q_width = self.config.num_attention_heads * head_dim;
857        let kv_width = self.config.num_key_value_heads * head_dim;
858        let intermediate = self.config.intermediate_size;
859
860        if let Some(weight) =
861            binder.take_shaped("embed_tokens.weight", &[self.config.vocab_size, hidden])?
862        {
863            self.embed_tokens.set_weight(weight)?;
864        }
865
866        for (i, layer) in self.layers.iter_mut().enumerate() {
867            let attn = format!("layers.{i}.self_attn");
868            if let Some(w) =
869                binder.take_shaped(&format!("{attn}.q_proj.weight"), &[q_width, hidden])?
870            {
871                layer.self_attn.q_proj.set_weight(w)?;
872            }
873            if let Some(w) =
874                binder.take_shaped(&format!("{attn}.k_proj.weight"), &[kv_width, hidden])?
875            {
876                layer.self_attn.k_proj.set_weight(w)?;
877            }
878            if let Some(w) =
879                binder.take_shaped(&format!("{attn}.v_proj.weight"), &[kv_width, hidden])?
880            {
881                layer.self_attn.v_proj.set_weight(w)?;
882            }
883            if let Some(w) =
884                binder.take_shaped(&format!("{attn}.o_proj.weight"), &[hidden, q_width])?
885            {
886                layer.self_attn.o_proj.set_weight(w)?;
887            }
888
889            let mlp = format!("layers.{i}.mlp");
890            if let Some(w) =
891                binder.take_shaped(&format!("{mlp}.gate_proj.weight"), &[intermediate, hidden])?
892            {
893                layer.mlp.gate_proj.set_weight(w)?;
894            }
895            if let Some(w) =
896                binder.take_shaped(&format!("{mlp}.up_proj.weight"), &[intermediate, hidden])?
897            {
898                layer.mlp.up_proj.set_weight(w)?;
899            }
900            if let Some(w) =
901                binder.take_shaped(&format!("{mlp}.down_proj.weight"), &[hidden, intermediate])?
902            {
903                layer.mlp.down_proj.set_weight(w)?;
904            }
905
906            if let Some(w) =
907                binder.take_shaped(&format!("layers.{i}.input_layernorm.weight"), &[hidden])?
908            {
909                layer.input_layernorm.set_weight(w)?;
910            }
911            if let Some(b) = binder.take_optional(&format!("layers.{i}.input_layernorm.bias")) {
912                layer.input_layernorm.set_bias(b)?;
913            }
914            // Cohere's decoder shares a single norm across the parallel
915            // attention/MLP block, so this parameter is genuinely absent from
916            // released checkpoints: request it optionally instead of reporting a
917            // spurious "missing" for every layer.
918            if let Some(w) =
919                binder.take_optional(&format!("layers.{i}.post_attention_layernorm.weight"))
920            {
921                layer.post_attention_layernorm.set_weight(w)?;
922            }
923            if let Some(b) =
924                binder.take_optional(&format!("layers.{i}.post_attention_layernorm.bias"))
925            {
926                layer.post_attention_layernorm.set_bias(b)?;
927            }
928        }
929
930        if let Some(w) = binder.take_shaped("norm.weight", &[hidden])? {
931            self.norm.set_weight(w)?;
932        }
933        if let Some(b) = binder.take_optional("norm.bias") {
934            self.norm.set_bias(b)?;
935        }
936
937        binder.finish(allowed_unused_prefixes)
938    }
939}
940
941/// Command R Model Output
942#[derive(Debug, Clone)]
943pub struct CommandRModelOutput {
944    pub last_hidden_state: Tensor,
945    pub past_key_values: Option<Vec<(Tensor, Tensor)>>,
946    pub hidden_states: Option<Vec<Tensor>>,
947    pub attentions: Option<Vec<Tensor>>,
948}
949
950/// Command R for Causal Language Modeling
951#[derive(Debug, Clone)]
952pub struct CommandRForCausalLM {
953    model: CommandRModel,
954    lm_head: Linear,
955    config: CommandRConfig,
956}
957
958impl CommandRForCausalLM {
959    pub fn new(config: &CommandRConfig) -> Result<Self> {
960        let model = CommandRModel::new(config)?;
961        let lm_head = Linear::new(config.hidden_size, config.vocab_size, config.use_bias);
962
963        Ok(Self {
964            model,
965            lm_head,
966            config: config.clone(),
967        })
968    }
969
970    pub fn forward(
971        &self,
972        input_ids: &Tensor,
973        attention_mask: Option<&Tensor>,
974        position_ids: Option<&Tensor>,
975        past_key_values: Option<&[(Tensor, Tensor)]>,
976        labels: Option<&Tensor>,
977    ) -> Result<CommandRCausalLMOutput> {
978        let outputs =
979            self.model.forward(input_ids, attention_mask, position_ids, past_key_values)?;
980
981        let logits = self.lm_head.forward(outputs.last_hidden_state)?;
982
983        let loss = match labels {
984            Some(labels) => Some(Self::causal_lm_loss(&logits, labels)?),
985            None => None,
986        };
987
988        Ok(CommandRCausalLMOutput {
989            loss,
990            logits,
991            past_key_values: outputs.past_key_values,
992            hidden_states: outputs.hidden_states,
993            attentions: outputs.attentions,
994        })
995    }
996
997    /// Shifted causal-LM cross-entropy loss.
998    ///
999    /// `logits` is `[batch, seq_len, vocab]` and `labels` holds `batch * seq_len`
1000    /// token ids. Position `t` predicts `labels[t + 1]`, so the loss is
1001    /// `mean(-log softmax(logits[t])[labels[t+1]])` over the batch. Labels equal
1002    /// to `-100` are ignored, matching the HuggingFace convention.
1003    fn causal_lm_loss(logits: &Tensor, labels: &Tensor) -> Result<Tensor> {
1004        let shape = logits.shape().to_vec();
1005        if shape.len() != 3 {
1006            return Err(tensor_op_error(
1007                "CommandRForCausalLM::causal_lm_loss",
1008                format!("expected logits [batch, seq_len, vocab], got {shape:?}"),
1009            ));
1010        }
1011        let (batch_size, seq_len, vocab_size) = (shape[0], shape[1], shape[2]);
1012
1013        let label_values = labels.data()?;
1014        if label_values.len() != batch_size * seq_len {
1015            return Err(tensor_op_error(
1016                "CommandRForCausalLM::causal_lm_loss",
1017                format!(
1018                    "expected {} labels, got {}",
1019                    batch_size * seq_len,
1020                    label_values.len()
1021                ),
1022            ));
1023        }
1024        if seq_len < 2 {
1025            return Err(tensor_op_error(
1026                "CommandRForCausalLM::causal_lm_loss",
1027                "causal-LM loss needs at least two positions to shift".to_string(),
1028            ));
1029        }
1030
1031        let logit_values = logits.data()?;
1032        let mut total = 0.0f32;
1033        let mut counted = 0usize;
1034        for b in 0..batch_size {
1035            for t in 0..seq_len - 1 {
1036                let target = label_values[b * seq_len + t + 1];
1037                if target < 0.0 {
1038                    continue; // ignore_index
1039                }
1040                let target_idx = target as usize;
1041                if target_idx >= vocab_size {
1042                    return Err(tensor_op_error(
1043                        "CommandRForCausalLM::causal_lm_loss",
1044                        format!("label {target_idx} is outside the vocabulary ({vocab_size})"),
1045                    ));
1046                }
1047                let row = &logit_values[(b * seq_len + t) * vocab_size..][..vocab_size];
1048                let max_logit = row.iter().copied().fold(f32::NEG_INFINITY, f32::max);
1049                let sum_exp: f32 = row.iter().map(|&x| (x - max_logit).exp()).sum();
1050                let log_prob = row[target_idx] - max_logit - sum_exp.ln();
1051                total -= log_prob;
1052                counted += 1;
1053            }
1054        }
1055
1056        if counted == 0 {
1057            return Err(tensor_op_error(
1058                "CommandRForCausalLM::causal_lm_loss",
1059                "every label was masked out; no loss could be computed".to_string(),
1060            ));
1061        }
1062        Tensor::scalar(total / counted as f32)
1063    }
1064
1065    /// Autoregressive generation with a real KV cache and real sampling.
1066    ///
1067    /// `temperature <= 0` selects greedy decoding; otherwise the logits are
1068    /// temperature-scaled and passed to top-k / top-p / full multinomial
1069    /// sampling from [`GenerationUtils`].
1070    pub fn generate(
1071        &mut self,
1072        input_ids: &Tensor,
1073        max_length: usize,
1074        temperature: f32,
1075        top_k: Option<usize>,
1076        top_p: Option<f32>,
1077    ) -> Result<Tensor> {
1078        let mut rng = thread_rng();
1079        self.generate_with_rng(input_ids, max_length, temperature, top_k, top_p, &mut rng)
1080    }
1081
1082    /// Same as [`generate`](Self::generate) but with a caller-supplied RNG so
1083    /// sampling can be made reproducible in tests.
1084    pub fn generate_with_rng(
1085        &mut self,
1086        input_ids: &Tensor,
1087        max_length: usize,
1088        temperature: f32,
1089        top_k: Option<usize>,
1090        top_p: Option<f32>,
1091        rng: &mut impl Rng,
1092    ) -> Result<Tensor> {
1093        let input_shape = input_ids.shape().to_vec();
1094        if input_shape.len() != 2 || input_shape[0] != 1 {
1095            return Err(tensor_op_error(
1096                "CommandRForCausalLM::generate",
1097                format!("generation expects input ids of shape [1, seq_len], got {input_shape:?}"),
1098            ));
1099        }
1100
1101        let mut token_ids: Vec<i64> = match input_ids {
1102            Tensor::I64(arr) => arr.iter().copied().collect(),
1103            other => other.data()?.iter().map(|&value| value as i64).collect(),
1104        };
1105        if token_ids.is_empty() {
1106            return Err(tensor_op_error(
1107                "CommandRForCausalLM::generate",
1108                "input_ids must not be empty".to_string(),
1109            ));
1110        }
1111
1112        let mut past_key_values: Option<Vec<(Tensor, Tensor)>> = None;
1113        let mut step_input = Tensor::from_vec_i64(token_ids.clone(), &[1, token_ids.len()])?;
1114
1115        for _ in 0..max_length {
1116            let outputs =
1117                self.forward(&step_input, None, None, past_key_values.as_deref(), None)?;
1118
1119            let logits_shape = outputs.logits.shape().to_vec();
1120            let vocab_size = logits_shape[logits_shape.len() - 1];
1121            let step_len = logits_shape[1];
1122            let all_logits = outputs.logits.data()?;
1123            let mut next_logits = all_logits[(step_len - 1) * vocab_size..][..vocab_size].to_vec();
1124
1125            if temperature > 0.0 {
1126                GenerationUtils::apply_temperature(&mut next_logits, temperature);
1127            }
1128            let next_token = Self::sample_next_token(&next_logits, temperature, top_k, top_p, rng)?;
1129
1130            token_ids.push(next_token as i64);
1131            past_key_values = outputs.past_key_values;
1132
1133            if let Some(eos_id) = self.config.eos_token_id {
1134                if next_token as usize == eos_id {
1135                    break;
1136                }
1137            }
1138
1139            // With a populated cache only the newly generated token is fed back.
1140            step_input = if past_key_values.is_some() {
1141                Tensor::from_vec_i64(vec![next_token as i64], &[1, 1])?
1142            } else {
1143                Tensor::from_vec_i64(token_ids.clone(), &[1, token_ids.len()])?
1144            };
1145        }
1146
1147        let generated_len = token_ids.len();
1148        Tensor::from_vec_i64(token_ids, &[1, generated_len])
1149    }
1150
1151    /// Pick the next token from (already temperature-scaled) logits.
1152    ///
1153    /// Delegates to the crate-wide [`GenerationUtils`] implementations instead of
1154    /// re-deriving them: `top_k` and `top_p` are honoured, and with neither set
1155    /// the token is drawn from the full softmax distribution. `temperature <= 0`
1156    /// means deterministic greedy decoding.
1157    fn sample_next_token(
1158        logits: &[f32],
1159        temperature: f32,
1160        top_k: Option<usize>,
1161        top_p: Option<f32>,
1162        rng: &mut impl Rng,
1163    ) -> Result<u32> {
1164        if logits.is_empty() {
1165            return Err(tensor_op_error(
1166                "CommandRForCausalLM::sample_next_token",
1167                "logits must not be empty".to_string(),
1168            ));
1169        }
1170        if temperature <= 0.0 {
1171            return Ok(GenerationUtils::sample_greedy(logits));
1172        }
1173        if let Some(k) = top_k {
1174            let k = k.clamp(1, logits.len());
1175            return GenerationUtils::sample_top_k(logits, k, rng);
1176        }
1177        if let Some(p) = top_p {
1178            return GenerationUtils::sample_top_p(logits, p, rng);
1179        }
1180        let probs = GenerationUtils::softmax(logits);
1181        Ok(GenerationUtils::sample_from_probs(&probs, rng)? as u32)
1182    }
1183}
1184
1185/// Command R Causal LM Output
1186#[derive(Debug, Clone)]
1187pub struct CommandRCausalLMOutput {
1188    pub loss: Option<Tensor>,
1189    pub logits: Tensor,
1190    pub past_key_values: Option<Vec<(Tensor, Tensor)>>,
1191    pub hidden_states: Option<Vec<Tensor>>,
1192    pub attentions: Option<Vec<Tensor>>,
1193}
1194
1195impl Model for CommandRForCausalLM {
1196    type Config = CommandRConfig;
1197    type Input = Tensor;
1198    type Output = Tensor;
1199
1200    fn forward(&self, input: Self::Input) -> Result<Self::Output> {
1201        // Forward through the model to get hidden states. The trait method takes
1202        // hidden states (the inherent `CommandRModel::forward` takes token ids),
1203        // so disambiguate explicitly.
1204        let hidden_states = <CommandRModel as Model>::forward(&self.model, input)?;
1205
1206        // Apply language modeling head to get logits
1207        let logits = self.lm_head.forward(hidden_states)?;
1208
1209        Ok(logits)
1210    }
1211
1212    /// Load a HuggingFace Command-R checkpoint into the base model and the head.
1213    ///
1214    /// Cohere checkpoints tie the LM head to the input embeddings, so a
1215    /// checkpoint without `lm_head.weight` reuses the embedding matrix — that is
1216    /// what the tied configuration means, not a fallback guess.
1217    fn load_pretrained(&mut self, reader: &mut dyn std::io::Read) -> Result<()> {
1218        let checkpoint = Checkpoint::from_reader(reader)?;
1219        self.model.load_checkpoint(&checkpoint, &["lm_head."])?;
1220
1221        let expected = [self.config.vocab_size, self.config.hidden_size];
1222        let head = match checkpoint.get("lm_head.weight") {
1223            Some(weight) => weight,
1224            None => {
1225                let embed_name = if checkpoint.contains("model.embed_tokens.weight") {
1226                    "model.embed_tokens.weight"
1227                } else {
1228                    "embed_tokens.weight"
1229                };
1230                checkpoint.get(embed_name).ok_or_else(|| {
1231                    tensor_op_error(
1232                        "CommandRForCausalLM::load_pretrained",
1233                        "checkpoint holds neither lm_head.weight nor an embedding matrix to tie it to"
1234                            .to_string(),
1235                    )
1236                })?
1237            },
1238        };
1239        if head.shape() != expected {
1240            return Err(tensor_op_error(
1241                "CommandRForCausalLM::load_pretrained",
1242                format!(
1243                    "language-model head has shape {:?} but this model expects {expected:?}",
1244                    head.shape()
1245                ),
1246            ));
1247        }
1248        self.lm_head.set_weight(head.clone())?;
1249        Ok(())
1250    }
1251
1252    fn get_config(&self) -> &Self::Config {
1253        &self.config
1254    }
1255
1256    fn num_parameters(&self) -> usize {
1257        self.model.num_parameters() + self.lm_head.parameter_count()
1258    }
1259}
1260
1261impl CommandRForCausalLM {
1262    /// Load model weights from a directory containing HuggingFace format weights
1263    pub fn load_from_path(&mut self, model_path: impl AsRef<std::path::Path>) -> Result<()> {
1264        use crate::weight_loading::WeightLoadingConfig;
1265
1266        let config = WeightLoadingConfig {
1267            lazy_loading: true,
1268            memory_mapped: false,
1269            ..Default::default()
1270        };
1271        self.load_from_path_with_config(model_path, config)
1272    }
1273
1274    /// Load model weights with an explicit [`WeightLoadingConfig`](crate::weight_loading::WeightLoadingConfig).
1275    pub fn load_from_path_with_config(
1276        &mut self,
1277        model_path: impl AsRef<std::path::Path>,
1278        config: crate::weight_loading::WeightLoadingConfig,
1279    ) -> Result<()> {
1280        use crate::weight_loading::auto_create_loader;
1281
1282        let mut loader = auto_create_loader(model_path, Some(config))?;
1283
1284        // Load embedding weights
1285        if let Ok(embed_weights) = loader.load_tensor("model.embed_tokens.weight") {
1286            self.model.embed_tokens.set_weight(embed_weights)?;
1287        }
1288
1289        // Load layer weights
1290        for (i, layer) in self.model.layers.iter_mut().enumerate() {
1291            // Load attention weights
1292            let attn_prefix = format!("model.layers.{}.self_attn", i);
1293
1294            if let Ok(q_weight) = loader.load_tensor(&format!("{}.q_proj.weight", attn_prefix)) {
1295                layer.self_attn.q_proj.set_weight(q_weight)?;
1296            }
1297            if let Ok(k_weight) = loader.load_tensor(&format!("{}.k_proj.weight", attn_prefix)) {
1298                layer.self_attn.k_proj.set_weight(k_weight)?;
1299            }
1300            if let Ok(v_weight) = loader.load_tensor(&format!("{}.v_proj.weight", attn_prefix)) {
1301                layer.self_attn.v_proj.set_weight(v_weight)?;
1302            }
1303            if let Ok(o_weight) = loader.load_tensor(&format!("{}.o_proj.weight", attn_prefix)) {
1304                layer.self_attn.o_proj.set_weight(o_weight)?;
1305            }
1306
1307            // Load MLP weights
1308            let mlp_prefix = format!("model.layers.{}.mlp", i);
1309
1310            if let Ok(gate_weight) = loader.load_tensor(&format!("{}.gate_proj.weight", mlp_prefix))
1311            {
1312                layer.mlp.gate_proj.set_weight(gate_weight)?;
1313            }
1314            if let Ok(up_weight) = loader.load_tensor(&format!("{}.up_proj.weight", mlp_prefix)) {
1315                layer.mlp.up_proj.set_weight(up_weight)?;
1316            }
1317            if let Ok(down_weight) = loader.load_tensor(&format!("{}.down_proj.weight", mlp_prefix))
1318            {
1319                layer.mlp.down_proj.set_weight(down_weight)?;
1320            }
1321
1322            // Load layer norm weights
1323            if let Ok(ln1_weight) =
1324                loader.load_tensor(&format!("model.layers.{}.input_layernorm.weight", i))
1325            {
1326                layer.input_layernorm.set_weight(ln1_weight)?;
1327            }
1328            if let Ok(ln2_weight) = loader.load_tensor(&format!(
1329                "model.layers.{}.post_attention_layernorm.weight",
1330                i
1331            )) {
1332                layer.post_attention_layernorm.set_weight(ln2_weight)?;
1333            }
1334        }
1335
1336        // Load final layer norm
1337        if let Ok(norm_weight) = loader.load_tensor("model.norm.weight") {
1338            self.model.norm.set_weight(norm_weight)?;
1339        }
1340
1341        // Load LM head weights
1342        if let Ok(lm_head_weight) = loader.load_tensor("lm_head.weight") {
1343            self.lm_head.set_weight(lm_head_weight)?;
1344        }
1345
1346        Ok(())
1347    }
1348
1349    /// Load from HuggingFace Hub model name
1350    pub fn load_from_huggingface(&mut self, model_name: &str) -> Result<()> {
1351        // Check if model is cached locally
1352        let cache_dir = std::env::var("HF_HOME")
1353            .or_else(|_| std::env::var("HUGGINGFACE_HUB_CACHE"))
1354            .unwrap_or_else(|_| {
1355                std::env::var("HOME").unwrap_or_else(|_| ".".to_string())
1356                    + "/.cache/huggingface/hub"
1357            });
1358
1359        let model_path = std::path::Path::new(&cache_dir)
1360            .join(format!("models--{}", model_name.replace("/", "--")));
1361
1362        if model_path.exists() {
1363            self.load_from_path(&model_path)
1364        } else {
1365            // Attempt to download the model from HuggingFace Hub
1366            self.download_from_huggingface_hub(model_name, &model_path)?;
1367            self.load_from_path(&model_path)
1368        }
1369    }
1370
1371    /// Download model from HuggingFace Hub
1372    fn download_from_huggingface_hub(
1373        &self,
1374        model_name: &str,
1375        model_path: &std::path::Path,
1376    ) -> Result<()> {
1377        use std::process::Command;
1378
1379        tracing::info!(
1380            "Downloading model {} from HuggingFace Hub to {:?}",
1381            model_name,
1382            model_path
1383        );
1384
1385        // Create the model directory
1386        std::fs::create_dir_all(model_path).map_err(|e| {
1387            trustformers_core::errors::TrustformersError::io_error(format!(
1388                "Failed to create model directory: {}",
1389                e
1390            ))
1391        })?;
1392
1393        // List of essential files for Command-R models
1394        let essential_files = vec![
1395            "config.json",
1396            "tokenizer.json",
1397            "tokenizer_config.json",
1398            "pytorch_model.bin", // Try .bin first
1399            "model.safetensors", // Fall back to safetensors
1400        ];
1401
1402        let base_url = format!("https://huggingface.co/{}/resolve/main", model_name);
1403
1404        // Try to download each essential file
1405        for file_name in &essential_files {
1406            let file_url = format!("{}/{}", base_url, file_name);
1407            let file_path = model_path.join(file_name);
1408
1409            tracing::info!("Attempting to download {}", file_url);
1410
1411            // Convert path to string once for both commands
1412            let file_path_str = file_path.to_str().ok_or_else(|| {
1413                TrustformersError::invalid_config(format!("Invalid UTF-8 in path: {:?}", file_path))
1414            })?;
1415
1416            // Try using curl first
1417            let curl_result = Command::new("curl")
1418                .args([
1419                    "-L", // Follow redirects
1420                    "-f", // Fail on HTTP errors
1421                    "-o",
1422                    file_path_str,
1423                    &file_url,
1424                ])
1425                .output();
1426
1427            match curl_result {
1428                Ok(output) if output.status.success() => {
1429                    tracing::info!("Successfully downloaded {}", file_name);
1430                    continue;
1431                },
1432                Ok(output) => {
1433                    tracing::warn!(
1434                        "Failed to download {} with curl: {}",
1435                        file_name,
1436                        String::from_utf8_lossy(&output.stderr)
1437                    );
1438                },
1439                Err(e) => {
1440                    tracing::info!("curl not available: {}", e);
1441                },
1442            }
1443
1444            // Try using wget as fallback
1445            let wget_result = Command::new("wget").args(["-O", file_path_str, &file_url]).output();
1446
1447            match wget_result {
1448                Ok(output) if output.status.success() => {
1449                    tracing::info!("Successfully downloaded {} with wget", file_name);
1450                    continue;
1451                },
1452                Ok(output) => {
1453                    tracing::warn!(
1454                        "Failed to download {} with wget: {}",
1455                        file_name,
1456                        String::from_utf8_lossy(&output.stderr)
1457                    );
1458                },
1459                Err(e) => {
1460                    tracing::info!("wget not available: {}", e);
1461                },
1462            }
1463
1464            // If essential files like config.json or pytorch_model.bin fail, return error
1465            if matches!(file_name, &"config.json" | &"pytorch_model.bin") {
1466                return Err(trustformers_core::errors::TrustformersError::io_error(format!(
1467                    "Failed to download essential file {} for model {}. Please ensure curl or wget is installed and you have internet access.",
1468                    file_name, model_name
1469                )));
1470            }
1471        }
1472
1473        tracing::info!(
1474            "Successfully downloaded model {} to {:?}",
1475            model_name,
1476            model_path
1477        );
1478        Ok(())
1479    }
1480
1481    /// Load weights through a **memory-mapped** loader.
1482    ///
1483    /// The loading config asks for a memory-mapped reader, so tensors are
1484    /// materialised one at a time out of the mapping instead of through an
1485    /// intermediate copy of the whole checkpoint. It is *not* deferred loading:
1486    /// when this call returns, every weight the model knows about is resident.
1487    /// Model parameters are owned `Tensor`s, so there is nothing left to resolve
1488    /// on first access.
1489    pub fn load_with_mmap(&mut self, model_path: impl AsRef<std::path::Path>) -> Result<()> {
1490        use crate::weight_loading::WeightLoadingConfig;
1491
1492        let config = WeightLoadingConfig {
1493            lazy_loading: true,
1494            memory_mapped: true,
1495            streaming: false,
1496            ..Default::default()
1497        };
1498        self.load_from_path_with_config(model_path, config)
1499    }
1500
1501    /// Deprecated alias for [`load_with_mmap`](Self::load_with_mmap).
1502    ///
1503    /// The old name promised on-demand tensor resolution that this loader has
1504    /// never performed; it loads every tensor eagerly.
1505    #[deprecated(
1506        since = "0.2.1",
1507        note = "renamed to `load_with_mmap`: loading is memory-mapped, not deferred"
1508    )]
1509    pub fn load_with_lazy_loading(
1510        &mut self,
1511        model_path: impl AsRef<std::path::Path>,
1512    ) -> Result<()> {
1513        self.load_with_mmap(model_path)
1514    }
1515}
1516
1517impl Config for CommandRConfig {
1518    fn validate(&self) -> Result<()> {
1519        self.validate().map_err(|e| invalid_config("config_validation", &e))
1520    }
1521
1522    fn architecture(&self) -> &'static str {
1523        "command-r"
1524    }
1525}
1526
1527#[cfg(test)]
1528#[path = "model_tests.rs"]
1529mod tests;