1use crate::command_r::config::CommandRConfig;
2use crate::common::ActivationType;
3use crate::generation_utils::GenerationUtils;
4use crate::weight_loading::{Checkpoint, LoadReport};
5use scirs2_core::random::{thread_rng, Rng}; use trustformers_core::{
7 errors::{invalid_config, tensor_op_error, Result, TrustformersError},
8 layers::{Embedding, LayerNorm, Linear},
9 tensor::Tensor,
10 traits::{Config, Layer, Model},
11};
12
13#[derive(Debug, Clone)]
19pub struct CommandRRoPE {
20 dim: usize,
21 max_seq_len: usize,
22 base: f32,
23 inv_freq: Vec<f32>,
25}
26
27impl CommandRRoPE {
28 pub fn new(dim: usize, max_seq_len: usize, base: f32) -> Result<Self> {
29 if dim < 2 || !dim.is_multiple_of(2) {
30 return Err(invalid_config(
31 "CommandRRoPE::new",
32 format!("head dimension must be even and >= 2, got {dim}"),
33 ));
34 }
35 let inv_freq = (0..dim)
36 .step_by(2)
37 .map(|i| 1.0 / base.powf(i as f32 / dim as f32))
38 .collect::<Vec<f32>>();
39
40 Ok(Self {
41 dim,
42 max_seq_len,
43 base,
44 inv_freq,
45 })
46 }
47
48 pub fn dim(&self) -> usize {
50 self.dim
51 }
52
53 pub fn max_seq_len(&self) -> usize {
55 self.max_seq_len
56 }
57
58 pub fn base(&self) -> f32 {
60 self.base
61 }
62
63 pub fn inv_freq(&self) -> &[f32] {
65 &self.inv_freq
66 }
67
68 pub fn cos_sin(&self, positions: &[usize]) -> Result<(Tensor, Tensor)> {
72 let half = self.dim / 2;
73 let mut cos_vals = Vec::with_capacity(positions.len() * half);
74 let mut sin_vals = Vec::with_capacity(positions.len() * half);
75 for &pos in positions {
76 for freq in &self.inv_freq {
77 let angle = pos as f32 * freq;
78 cos_vals.push(angle.cos());
79 sin_vals.push(angle.sin());
80 }
81 }
82 Ok((
83 Tensor::from_vec(cos_vals, &[positions.len(), half])?,
84 Tensor::from_vec(sin_vals, &[positions.len(), half])?,
85 ))
86 }
87
88 pub fn rotate_in_place(
93 &self,
94 data: &mut [f32],
95 batch: usize,
96 num_heads: usize,
97 head_dim: usize,
98 positions: &[usize],
99 ) -> Result<()> {
100 if head_dim != self.dim {
101 return Err(tensor_op_error(
102 "CommandRRoPE::rotate_in_place",
103 format!("head_dim {head_dim} does not match RoPE dim {}", self.dim),
104 ));
105 }
106 let seq_len = positions.len();
107 let expected = batch * seq_len * num_heads * head_dim;
108 if data.len() != expected {
109 return Err(tensor_op_error(
110 "CommandRRoPE::rotate_in_place",
111 format!("expected {expected} elements, got {}", data.len()),
112 ));
113 }
114 if let Some(&max_pos) = positions.iter().max() {
115 if max_pos >= self.max_seq_len {
116 return Err(tensor_op_error(
117 "CommandRRoPE::rotate_in_place",
118 format!(
119 "position {max_pos} exceeds max_sequence_length {}",
120 self.max_seq_len
121 ),
122 ));
123 }
124 }
125
126 let half = head_dim / 2;
127 for b in 0..batch {
128 for (t, &pos) in positions.iter().enumerate() {
129 for head in 0..num_heads {
130 let base = ((b * seq_len + t) * num_heads + head) * head_dim;
131 for i in 0..half {
132 let angle = pos as f32 * self.inv_freq[i];
133 let (sin_val, cos_val) = angle.sin_cos();
134 let x0 = data[base + i];
135 let x1 = data[base + i + half];
136 data[base + i] = x0 * cos_val - x1 * sin_val;
137 data[base + i + half] = x0 * sin_val + x1 * cos_val;
138 }
139 }
140 }
141 }
142 Ok(())
143 }
144}
145
146#[derive(Debug, Clone)]
148pub struct CommandRAttention {
149 #[allow(dead_code)]
150 config: CommandRConfig,
151 hidden_size: usize,
152 num_heads: usize,
153 num_key_value_heads: usize,
154 head_dim: usize,
155
156 q_proj: Linear,
157 k_proj: Linear,
158 v_proj: Linear,
159 o_proj: Linear,
160
161 rope: CommandRRoPE,
162 attention_dropout: f32,
163 #[allow(dead_code)]
164 use_flash_attention: bool,
165}
166
167impl CommandRAttention {
168 pub fn new(config: &CommandRConfig) -> Result<Self> {
169 let hidden_size = config.hidden_size;
170 let num_heads = config.num_attention_heads;
171 let num_key_value_heads = config.num_key_value_heads;
172 let head_dim = config.head_dim();
173
174 let q_proj = Linear::new(hidden_size, num_heads * head_dim, config.use_bias);
175 let k_proj = Linear::new(hidden_size, num_key_value_heads * head_dim, config.use_bias);
176 let v_proj = Linear::new(hidden_size, num_key_value_heads * head_dim, config.use_bias);
177 let o_proj = Linear::new(num_heads * head_dim, hidden_size, config.use_bias);
178
179 let rope = CommandRRoPE::new(head_dim, config.max_sequence_length, config.rope_theta)?;
180
181 Ok(Self {
182 config: config.clone(),
183 hidden_size,
184 num_heads,
185 num_key_value_heads,
186 head_dim,
187 q_proj,
188 k_proj,
189 v_proj,
190 o_proj,
191 rope,
192 attention_dropout: config.attention_dropout,
193 use_flash_attention: config.use_flash_attention,
194 })
195 }
196
197 pub fn forward(
212 &self,
213 hidden_states: &Tensor,
214 attention_mask: Option<&Tensor>,
215 position_ids: &Tensor,
216 past_key_value: Option<(&Tensor, &Tensor)>,
217 ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
218 let shape = hidden_states.shape().to_vec();
219 if shape.len() != 3 {
220 return Err(tensor_op_error(
221 "CommandRAttention::forward",
222 format!("expected [batch, seq_len, hidden_size], got {shape:?}"),
223 ));
224 }
225 let (batch_size, seq_len, hidden) = (shape[0], shape[1], shape[2]);
226 if hidden != self.hidden_size {
227 return Err(tensor_op_error(
228 "CommandRAttention::forward",
229 format!(
230 "input hidden size {hidden} does not match config hidden size {}",
231 self.hidden_size
232 ),
233 ));
234 }
235
236 let past_len = match past_key_value {
238 Some((past_key, past_value)) => {
239 let key_shape = past_key.shape().to_vec();
240 let value_shape = past_value.shape().to_vec();
241 let expected_trailing = [batch_size, self.num_key_value_heads, self.head_dim];
242 let valid = |s: &[usize]| {
243 s.len() == 4
244 && s[0] == expected_trailing[0]
245 && s[2] == expected_trailing[1]
246 && s[3] == expected_trailing[2]
247 };
248 if !valid(&key_shape) || !valid(&value_shape) || key_shape[1] != value_shape[1] {
249 return Err(tensor_op_error(
250 "CommandRAttention::forward",
251 format!(
252 "past key/value must be [{}, past_len, {}, {}], got {key_shape:?} / {value_shape:?}",
253 batch_size, self.num_key_value_heads, self.head_dim
254 ),
255 ));
256 }
257 key_shape[1]
258 },
259 None => 0,
260 };
261
262 let positions = Self::resolve_positions(position_ids, seq_len, past_len);
264
265 let query_states = self.q_proj.forward(hidden_states.clone())?;
267 let key_states = self.k_proj.forward(hidden_states.clone())?;
268 let value_states = self.v_proj.forward(hidden_states.clone())?;
269
270 let mut query_data = query_states.data()?;
273 let mut key_data = key_states.data()?;
274 self.rope.rotate_in_place(
275 &mut query_data,
276 batch_size,
277 self.num_heads,
278 self.head_dim,
279 &positions,
280 )?;
281 self.rope.rotate_in_place(
282 &mut key_data,
283 batch_size,
284 self.num_key_value_heads,
285 self.head_dim,
286 &positions,
287 )?;
288
289 let new_key = Tensor::from_vec(
290 key_data,
291 &[batch_size, seq_len, self.num_key_value_heads, self.head_dim],
292 )?;
293 let new_value = value_states.reshape(&[
294 batch_size,
295 seq_len,
296 self.num_key_value_heads,
297 self.head_dim,
298 ])?;
299
300 let (key_cache, value_cache) = match past_key_value {
302 Some((past_key, past_value)) => (
303 Tensor::concat(&[past_key.clone(), new_key], 1)?,
304 Tensor::concat(&[past_value.clone(), new_value], 1)?,
305 ),
306 None => (new_key, new_value),
307 };
308
309 let total_len = past_len + seq_len;
310 let attn_output = self.scaled_dot_product_attention(
311 &query_data,
312 &key_cache,
313 &value_cache,
314 batch_size,
315 seq_len,
316 total_len,
317 past_len,
318 attention_mask,
319 )?;
320
321 let attn_output = attn_output.reshape(&[batch_size, seq_len, self.hidden_size])?;
322 let attn_output = self.o_proj.forward(attn_output)?;
323
324 Ok((attn_output, Some((key_cache, value_cache))))
325 }
326
327 fn resolve_positions(position_ids: &Tensor, seq_len: usize, past_len: usize) -> Vec<usize> {
332 if let Ok(values) = position_ids.data() {
333 if values.len() == seq_len {
334 return values.iter().map(|&p| p.max(0.0) as usize).collect();
335 }
336 if values.len() > seq_len && values.len() % seq_len == 0 {
337 return values[..seq_len].iter().map(|&p| p.max(0.0) as usize).collect();
339 }
340 }
341 (past_len..past_len + seq_len).collect()
342 }
343
344 #[allow(clippy::too_many_arguments)]
354 fn scaled_dot_product_attention(
355 &self,
356 query_data: &[f32],
357 key_cache: &Tensor,
358 value_cache: &Tensor,
359 batch_size: usize,
360 seq_len: usize,
361 total_len: usize,
362 past_len: usize,
363 attention_mask: Option<&Tensor>,
364 ) -> Result<Tensor> {
365 let head_dim = self.head_dim;
366 let num_heads = self.num_heads;
367 let num_kv_heads = self.num_key_value_heads;
368 if num_kv_heads == 0 || !num_heads.is_multiple_of(num_kv_heads) {
369 return Err(invalid_config(
370 "CommandRAttention",
371 format!(
372 "num_attention_heads {num_heads} must be a multiple of num_key_value_heads {num_kv_heads}"
373 ),
374 ));
375 }
376 let groups = num_heads / num_kv_heads;
377
378 let key_data = key_cache.data()?;
379 let value_data = value_cache.data()?;
380
381 let mask_data = match attention_mask {
383 Some(mask) => Some(mask.data()?),
384 None => None,
385 };
386 let mask_len = mask_data.as_ref().map(|m| m.len()).unwrap_or(0);
387
388 let scale = 1.0 / (head_dim as f32).sqrt();
389 let mut output = vec![0.0f32; batch_size * seq_len * num_heads * head_dim];
390 let mut scores = vec![0.0f32; total_len];
391
392 for b in 0..batch_size {
393 for head in 0..num_heads {
394 let kv_head = head / groups;
395 for q_idx in 0..seq_len {
396 let q_base = ((b * seq_len + q_idx) * num_heads + head) * head_dim;
397 let visible = past_len + q_idx + 1;
400 let mut max_score = f32::NEG_INFINITY;
401 for (key_idx, score) in scores.iter_mut().take(visible).enumerate() {
402 let k_base =
403 ((b * total_len + key_idx) * num_kv_heads + kv_head) * head_dim;
404 let mut dot = 0.0f32;
405 for d in 0..head_dim {
406 dot += query_data[q_base + d] * key_data[k_base + d];
407 }
408 dot *= scale;
409 if let Some(mask) = &mask_data {
410 dot += Self::mask_value(
411 mask, mask_len, b, head, q_idx, key_idx, batch_size, num_heads,
412 seq_len, total_len,
413 );
414 }
415 *score = dot;
416 if dot > max_score {
417 max_score = dot;
418 }
419 }
420
421 let mut sum = 0.0f32;
422 for score in scores.iter_mut().take(visible) {
423 *score = (*score - max_score).exp();
424 sum += *score;
425 }
426 let inv_sum = if sum > 0.0 { 1.0 / sum } else { 0.0 };
427
428 let out_base = ((b * seq_len + q_idx) * num_heads + head) * head_dim;
429 for (key_idx, score) in scores.iter().take(visible).enumerate() {
430 let weight = score * inv_sum;
431 if weight == 0.0 {
432 continue;
433 }
434 let v_base =
435 ((b * total_len + key_idx) * num_kv_heads + kv_head) * head_dim;
436 for d in 0..head_dim {
437 output[out_base + d] += weight * value_data[v_base + d];
438 }
439 }
440 }
441 }
442 }
443
444 Tensor::from_vec(output, &[batch_size, seq_len, num_heads, head_dim])
445 }
446
447 #[allow(clippy::too_many_arguments)]
450 fn mask_value(
451 mask: &[f32],
452 mask_len: usize,
453 batch: usize,
454 head: usize,
455 q_idx: usize,
456 key_idx: usize,
457 batch_size: usize,
458 num_heads: usize,
459 seq_len: usize,
460 total_len: usize,
461 ) -> f32 {
462 let plane = seq_len * total_len;
463 if mask_len == plane {
464 return mask[q_idx * total_len + key_idx];
465 }
466 if mask_len == batch_size * plane {
467 return mask[batch * plane + q_idx * total_len + key_idx];
468 }
469 if mask_len == batch_size * num_heads * plane {
470 return mask[((batch * num_heads + head) * seq_len + q_idx) * total_len + key_idx];
471 }
472 0.0
473 }
474
475 pub fn attention_dropout(&self) -> f32 {
480 self.attention_dropout
481 }
482
483 pub fn parameter_count(&self) -> usize {
484 self.q_proj.parameter_count()
485 + self.k_proj.parameter_count()
486 + self.v_proj.parameter_count()
487 + self.o_proj.parameter_count()
488 }
489}
490
491#[derive(Debug, Clone)]
493pub struct CommandRMLP {
494 #[allow(dead_code)]
495 config: CommandRConfig,
496 #[allow(dead_code)]
497 hidden_size: usize,
498 #[allow(dead_code)]
499 intermediate_size: usize,
500
501 gate_proj: Linear,
502 up_proj: Linear,
503 down_proj: Linear,
504
505 activation: ActivationType,
506}
507
508impl CommandRMLP {
509 pub fn new(config: &CommandRConfig) -> Result<Self> {
510 let hidden_size = config.hidden_size;
511 let intermediate_size = config.intermediate_size;
512
513 let gate_proj = Linear::new(hidden_size, intermediate_size, config.use_bias);
514 let up_proj = Linear::new(hidden_size, intermediate_size, config.use_bias);
515 let down_proj = Linear::new(intermediate_size, hidden_size, config.use_bias);
516
517 Ok(Self {
518 config: config.clone(),
519 hidden_size,
520 intermediate_size,
521 gate_proj,
522 up_proj,
523 down_proj,
524 activation: ActivationType::from_config_str_or(
527 &config.activation_function,
528 ActivationType::Gelu,
529 ),
530 })
531 }
532
533 pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
534 let gate_output = self.gate_proj.forward(x.clone())?;
536 let gate_output = self.activation.apply(&gate_output)?;
537
538 let up_output = self.up_proj.forward(x.clone())?;
540
541 let intermediate = gate_output.mul(&up_output)?;
543
544 let output = self.down_proj.forward(intermediate)?;
546
547 Ok(output)
548 }
549
550 pub fn parameter_count(&self) -> usize {
551 self.gate_proj.parameter_count()
552 + self.up_proj.parameter_count()
553 + self.down_proj.parameter_count()
554 }
555}
556
557#[derive(Debug, Clone)]
559pub struct CommandRDecoderLayer {
560 #[allow(dead_code)]
561 config: CommandRConfig,
562 #[allow(dead_code)]
563 hidden_size: usize,
564
565 self_attn: CommandRAttention,
566 mlp: CommandRMLP,
567 input_layernorm: LayerNorm,
568 post_attention_layernorm: LayerNorm,
569}
570
571impl CommandRDecoderLayer {
572 pub fn new(config: &CommandRConfig) -> Result<Self> {
573 let hidden_size = config.hidden_size;
574
575 let self_attn = CommandRAttention::new(config)?;
576 let mlp = CommandRMLP::new(config)?;
577
578 let input_layernorm = LayerNorm::new(vec![hidden_size], config.rms_norm_eps)?;
579 let post_attention_layernorm = LayerNorm::new(vec![hidden_size], config.rms_norm_eps)?;
580
581 Ok(Self {
582 config: config.clone(),
583 hidden_size,
584 self_attn,
585 mlp,
586 input_layernorm,
587 post_attention_layernorm,
588 })
589 }
590
591 pub fn forward(
592 &self,
593 hidden_states: &Tensor,
594 attention_mask: Option<&Tensor>,
595 position_ids: &Tensor,
596 past_key_value: Option<(&Tensor, &Tensor)>,
597 ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
598 let residual = hidden_states.clone();
599
600 let hidden_states = self.input_layernorm.forward(hidden_states.clone())?;
602
603 let (attn_output, present_key_value) =
605 self.self_attn
606 .forward(&hidden_states, attention_mask, position_ids, past_key_value)?;
607
608 let hidden_states = residual.add(&attn_output)?;
610 let residual = hidden_states.clone();
611
612 let hidden_states = self.post_attention_layernorm.forward(hidden_states)?;
614
615 let mlp_output = self.mlp.forward(&hidden_states)?;
617
618 let hidden_states = residual.add(&mlp_output)?;
620
621 Ok((hidden_states, present_key_value))
622 }
623
624 pub fn parameter_count(&self) -> usize {
625 self.self_attn.parameter_count()
626 + self.mlp.parameter_count()
627 + self.input_layernorm.parameter_count()
628 + self.post_attention_layernorm.parameter_count()
629 }
630}
631
632#[derive(Debug, Clone)]
634pub struct CommandRModel {
635 config: CommandRConfig,
636 #[allow(dead_code)]
637 vocab_size: usize,
638 #[allow(dead_code)]
639 hidden_size: usize,
640 #[allow(dead_code)]
641 num_hidden_layers: usize,
642
643 embed_tokens: Embedding,
644 layers: Vec<CommandRDecoderLayer>,
645 norm: LayerNorm,
646
647 #[allow(dead_code)]
648 pad_token_id: Option<usize>,
649 #[allow(dead_code)]
650 bos_token_id: Option<usize>,
651 #[allow(dead_code)]
652 eos_token_id: Option<usize>,
653}
654
655impl CommandRModel {
656 pub fn new(config: &CommandRConfig) -> Result<Self> {
657 config.validate().map_err(|e| invalid_config("config_validation", &e))?;
658
659 let vocab_size = config.vocab_size;
660 let hidden_size = config.hidden_size;
661 let num_hidden_layers = config.num_hidden_layers;
662
663 let embed_tokens = Embedding::new(vocab_size, hidden_size, None)?;
664
665 let mut layers = Vec::new();
666 for _ in 0..num_hidden_layers {
667 layers.push(CommandRDecoderLayer::new(config)?);
668 }
669
670 let norm = LayerNorm::new(vec![hidden_size], config.rms_norm_eps)?;
671
672 Ok(Self {
673 config: config.clone(),
674 vocab_size,
675 hidden_size,
676 num_hidden_layers,
677 embed_tokens,
678 layers,
679 norm,
680 pad_token_id: config.pad_token_id,
681 bos_token_id: config.bos_token_id,
682 eos_token_id: config.eos_token_id,
683 })
684 }
685
686 pub fn forward(
687 &self,
688 input_ids: &Tensor,
689 attention_mask: Option<&Tensor>,
690 position_ids: Option<&Tensor>,
691 past_key_values: Option<&[(Tensor, Tensor)]>,
692 ) -> Result<CommandRModelOutput> {
693 let input_shape = input_ids.shape().to_vec();
694 if input_shape.len() != 2 {
695 return Err(tensor_op_error(
696 "CommandRModel::forward",
697 format!("expected input ids of shape [batch, seq_len], got {input_shape:?}"),
698 ));
699 }
700 let batch_size = input_shape[0];
701 let seq_len = input_shape[1];
702
703 let past_len = past_key_values
705 .and_then(|pkv| pkv.first())
706 .map(|(key, _)| {
707 let shape = key.shape().to_vec();
708 if shape.len() == 4 {
709 shape[1]
710 } else {
711 0
712 }
713 })
714 .unwrap_or(0);
715
716 let position_ids = if let Some(pos_ids) = position_ids {
718 pos_ids.clone()
719 } else {
720 let pos_ids: Vec<f32> =
721 (past_len..past_len + seq_len).map(|position| position as f32).collect();
722 Tensor::from_vec(pos_ids, &[1, seq_len])?
723 };
724
725 let input_ids_vec = match input_ids {
728 Tensor::I64(arr) => arr.iter().map(|&x| x as u32).collect::<Vec<u32>>(),
729 _ => {
730 return Err(tensor_op_error(
731 "CommandRModel::forward",
732 "Input IDs must be integer tensor",
733 ))
734 },
735 };
736 let embedded = self.embed_tokens.forward(input_ids_vec)?;
738 let mut hidden_states =
739 embedded.reshape(&[batch_size, seq_len, self.config.hidden_size])?;
740
741 let mut present_key_values = Vec::new();
743 for (layer_idx, layer) in self.layers.iter().enumerate() {
744 let past_key_value = past_key_values.map(|pkv| (&pkv[layer_idx].0, &pkv[layer_idx].1));
745
746 let (layer_output, present_key_value) = layer.forward(
747 &hidden_states,
748 attention_mask,
749 &position_ids,
750 past_key_value,
751 )?;
752
753 hidden_states = layer_output;
754 if let Some(pkv) = present_key_value {
755 present_key_values.push(pkv);
756 }
757 }
758
759 let hidden_states = self.norm.forward(hidden_states)?;
761
762 Ok(CommandRModelOutput {
763 last_hidden_state: hidden_states,
764 past_key_values: if present_key_values.is_empty() {
765 None
766 } else {
767 Some(present_key_values)
768 },
769 hidden_states: None,
770 attentions: None,
771 })
772 }
773}
774
775impl Model for CommandRModel {
776 type Config = CommandRConfig;
777 type Input = Tensor;
778 type Output = Tensor;
779
780 fn forward(&self, input: Self::Input) -> Result<Self::Output> {
781 let mut hidden_states = input;
783
784 let seq_len = hidden_states.shape().get(1).copied().unwrap_or(0);
787 let position_ids = Tensor::from_vec(
788 (0..seq_len).map(|position| position as f32).collect(),
789 &[1, seq_len],
790 )?;
791 for layer in &self.layers {
792 let (new_hidden_states, _) = layer.forward(
793 &hidden_states,
794 None, &position_ids,
796 None, )?;
798 hidden_states = new_hidden_states;
799 }
800
801 hidden_states = self.norm.forward(hidden_states)?;
803
804 Ok(hidden_states)
805 }
806
807 fn load_pretrained(&mut self, reader: &mut dyn std::io::Read) -> Result<()> {
815 let checkpoint = Checkpoint::from_reader(reader)?;
816 self.load_checkpoint(&checkpoint, &["lm_head."])?;
817 Ok(())
818 }
819
820 fn get_config(&self) -> &Self::Config {
821 &self.config
822 }
823
824 fn num_parameters(&self) -> usize {
825 let embed_params = self.embed_tokens.parameter_count();
826 let layers_params: usize = self.layers.iter().map(|layer| layer.parameter_count()).sum();
827 let norm_params = self.norm.parameter_count();
828
829 embed_params + layers_params + norm_params
830 }
831}
832
833impl CommandRModel {
834 pub fn load_checkpoint(
847 &mut self,
848 checkpoint: &Checkpoint,
849 allowed_unused_prefixes: &[&str],
850 ) -> Result<LoadReport> {
851 let prefix = checkpoint.detect_prefix(&["model.", ""], "embed_tokens.weight")?;
852 let mut binder = checkpoint.binder(&prefix);
853
854 let hidden = self.config.hidden_size;
855 let head_dim = self.config.head_dim();
856 let q_width = self.config.num_attention_heads * head_dim;
857 let kv_width = self.config.num_key_value_heads * head_dim;
858 let intermediate = self.config.intermediate_size;
859
860 if let Some(weight) =
861 binder.take_shaped("embed_tokens.weight", &[self.config.vocab_size, hidden])?
862 {
863 self.embed_tokens.set_weight(weight)?;
864 }
865
866 for (i, layer) in self.layers.iter_mut().enumerate() {
867 let attn = format!("layers.{i}.self_attn");
868 if let Some(w) =
869 binder.take_shaped(&format!("{attn}.q_proj.weight"), &[q_width, hidden])?
870 {
871 layer.self_attn.q_proj.set_weight(w)?;
872 }
873 if let Some(w) =
874 binder.take_shaped(&format!("{attn}.k_proj.weight"), &[kv_width, hidden])?
875 {
876 layer.self_attn.k_proj.set_weight(w)?;
877 }
878 if let Some(w) =
879 binder.take_shaped(&format!("{attn}.v_proj.weight"), &[kv_width, hidden])?
880 {
881 layer.self_attn.v_proj.set_weight(w)?;
882 }
883 if let Some(w) =
884 binder.take_shaped(&format!("{attn}.o_proj.weight"), &[hidden, q_width])?
885 {
886 layer.self_attn.o_proj.set_weight(w)?;
887 }
888
889 let mlp = format!("layers.{i}.mlp");
890 if let Some(w) =
891 binder.take_shaped(&format!("{mlp}.gate_proj.weight"), &[intermediate, hidden])?
892 {
893 layer.mlp.gate_proj.set_weight(w)?;
894 }
895 if let Some(w) =
896 binder.take_shaped(&format!("{mlp}.up_proj.weight"), &[intermediate, hidden])?
897 {
898 layer.mlp.up_proj.set_weight(w)?;
899 }
900 if let Some(w) =
901 binder.take_shaped(&format!("{mlp}.down_proj.weight"), &[hidden, intermediate])?
902 {
903 layer.mlp.down_proj.set_weight(w)?;
904 }
905
906 if let Some(w) =
907 binder.take_shaped(&format!("layers.{i}.input_layernorm.weight"), &[hidden])?
908 {
909 layer.input_layernorm.set_weight(w)?;
910 }
911 if let Some(b) = binder.take_optional(&format!("layers.{i}.input_layernorm.bias")) {
912 layer.input_layernorm.set_bias(b)?;
913 }
914 if let Some(w) =
919 binder.take_optional(&format!("layers.{i}.post_attention_layernorm.weight"))
920 {
921 layer.post_attention_layernorm.set_weight(w)?;
922 }
923 if let Some(b) =
924 binder.take_optional(&format!("layers.{i}.post_attention_layernorm.bias"))
925 {
926 layer.post_attention_layernorm.set_bias(b)?;
927 }
928 }
929
930 if let Some(w) = binder.take_shaped("norm.weight", &[hidden])? {
931 self.norm.set_weight(w)?;
932 }
933 if let Some(b) = binder.take_optional("norm.bias") {
934 self.norm.set_bias(b)?;
935 }
936
937 binder.finish(allowed_unused_prefixes)
938 }
939}
940
941#[derive(Debug, Clone)]
943pub struct CommandRModelOutput {
944 pub last_hidden_state: Tensor,
945 pub past_key_values: Option<Vec<(Tensor, Tensor)>>,
946 pub hidden_states: Option<Vec<Tensor>>,
947 pub attentions: Option<Vec<Tensor>>,
948}
949
950#[derive(Debug, Clone)]
952pub struct CommandRForCausalLM {
953 model: CommandRModel,
954 lm_head: Linear,
955 config: CommandRConfig,
956}
957
958impl CommandRForCausalLM {
959 pub fn new(config: &CommandRConfig) -> Result<Self> {
960 let model = CommandRModel::new(config)?;
961 let lm_head = Linear::new(config.hidden_size, config.vocab_size, config.use_bias);
962
963 Ok(Self {
964 model,
965 lm_head,
966 config: config.clone(),
967 })
968 }
969
970 pub fn forward(
971 &self,
972 input_ids: &Tensor,
973 attention_mask: Option<&Tensor>,
974 position_ids: Option<&Tensor>,
975 past_key_values: Option<&[(Tensor, Tensor)]>,
976 labels: Option<&Tensor>,
977 ) -> Result<CommandRCausalLMOutput> {
978 let outputs =
979 self.model.forward(input_ids, attention_mask, position_ids, past_key_values)?;
980
981 let logits = self.lm_head.forward(outputs.last_hidden_state)?;
982
983 let loss = match labels {
984 Some(labels) => Some(Self::causal_lm_loss(&logits, labels)?),
985 None => None,
986 };
987
988 Ok(CommandRCausalLMOutput {
989 loss,
990 logits,
991 past_key_values: outputs.past_key_values,
992 hidden_states: outputs.hidden_states,
993 attentions: outputs.attentions,
994 })
995 }
996
997 fn causal_lm_loss(logits: &Tensor, labels: &Tensor) -> Result<Tensor> {
1004 let shape = logits.shape().to_vec();
1005 if shape.len() != 3 {
1006 return Err(tensor_op_error(
1007 "CommandRForCausalLM::causal_lm_loss",
1008 format!("expected logits [batch, seq_len, vocab], got {shape:?}"),
1009 ));
1010 }
1011 let (batch_size, seq_len, vocab_size) = (shape[0], shape[1], shape[2]);
1012
1013 let label_values = labels.data()?;
1014 if label_values.len() != batch_size * seq_len {
1015 return Err(tensor_op_error(
1016 "CommandRForCausalLM::causal_lm_loss",
1017 format!(
1018 "expected {} labels, got {}",
1019 batch_size * seq_len,
1020 label_values.len()
1021 ),
1022 ));
1023 }
1024 if seq_len < 2 {
1025 return Err(tensor_op_error(
1026 "CommandRForCausalLM::causal_lm_loss",
1027 "causal-LM loss needs at least two positions to shift".to_string(),
1028 ));
1029 }
1030
1031 let logit_values = logits.data()?;
1032 let mut total = 0.0f32;
1033 let mut counted = 0usize;
1034 for b in 0..batch_size {
1035 for t in 0..seq_len - 1 {
1036 let target = label_values[b * seq_len + t + 1];
1037 if target < 0.0 {
1038 continue; }
1040 let target_idx = target as usize;
1041 if target_idx >= vocab_size {
1042 return Err(tensor_op_error(
1043 "CommandRForCausalLM::causal_lm_loss",
1044 format!("label {target_idx} is outside the vocabulary ({vocab_size})"),
1045 ));
1046 }
1047 let row = &logit_values[(b * seq_len + t) * vocab_size..][..vocab_size];
1048 let max_logit = row.iter().copied().fold(f32::NEG_INFINITY, f32::max);
1049 let sum_exp: f32 = row.iter().map(|&x| (x - max_logit).exp()).sum();
1050 let log_prob = row[target_idx] - max_logit - sum_exp.ln();
1051 total -= log_prob;
1052 counted += 1;
1053 }
1054 }
1055
1056 if counted == 0 {
1057 return Err(tensor_op_error(
1058 "CommandRForCausalLM::causal_lm_loss",
1059 "every label was masked out; no loss could be computed".to_string(),
1060 ));
1061 }
1062 Tensor::scalar(total / counted as f32)
1063 }
1064
1065 pub fn generate(
1071 &mut self,
1072 input_ids: &Tensor,
1073 max_length: usize,
1074 temperature: f32,
1075 top_k: Option<usize>,
1076 top_p: Option<f32>,
1077 ) -> Result<Tensor> {
1078 let mut rng = thread_rng();
1079 self.generate_with_rng(input_ids, max_length, temperature, top_k, top_p, &mut rng)
1080 }
1081
1082 pub fn generate_with_rng(
1085 &mut self,
1086 input_ids: &Tensor,
1087 max_length: usize,
1088 temperature: f32,
1089 top_k: Option<usize>,
1090 top_p: Option<f32>,
1091 rng: &mut impl Rng,
1092 ) -> Result<Tensor> {
1093 let input_shape = input_ids.shape().to_vec();
1094 if input_shape.len() != 2 || input_shape[0] != 1 {
1095 return Err(tensor_op_error(
1096 "CommandRForCausalLM::generate",
1097 format!("generation expects input ids of shape [1, seq_len], got {input_shape:?}"),
1098 ));
1099 }
1100
1101 let mut token_ids: Vec<i64> = match input_ids {
1102 Tensor::I64(arr) => arr.iter().copied().collect(),
1103 other => other.data()?.iter().map(|&value| value as i64).collect(),
1104 };
1105 if token_ids.is_empty() {
1106 return Err(tensor_op_error(
1107 "CommandRForCausalLM::generate",
1108 "input_ids must not be empty".to_string(),
1109 ));
1110 }
1111
1112 let mut past_key_values: Option<Vec<(Tensor, Tensor)>> = None;
1113 let mut step_input = Tensor::from_vec_i64(token_ids.clone(), &[1, token_ids.len()])?;
1114
1115 for _ in 0..max_length {
1116 let outputs =
1117 self.forward(&step_input, None, None, past_key_values.as_deref(), None)?;
1118
1119 let logits_shape = outputs.logits.shape().to_vec();
1120 let vocab_size = logits_shape[logits_shape.len() - 1];
1121 let step_len = logits_shape[1];
1122 let all_logits = outputs.logits.data()?;
1123 let mut next_logits = all_logits[(step_len - 1) * vocab_size..][..vocab_size].to_vec();
1124
1125 if temperature > 0.0 {
1126 GenerationUtils::apply_temperature(&mut next_logits, temperature);
1127 }
1128 let next_token = Self::sample_next_token(&next_logits, temperature, top_k, top_p, rng)?;
1129
1130 token_ids.push(next_token as i64);
1131 past_key_values = outputs.past_key_values;
1132
1133 if let Some(eos_id) = self.config.eos_token_id {
1134 if next_token as usize == eos_id {
1135 break;
1136 }
1137 }
1138
1139 step_input = if past_key_values.is_some() {
1141 Tensor::from_vec_i64(vec![next_token as i64], &[1, 1])?
1142 } else {
1143 Tensor::from_vec_i64(token_ids.clone(), &[1, token_ids.len()])?
1144 };
1145 }
1146
1147 let generated_len = token_ids.len();
1148 Tensor::from_vec_i64(token_ids, &[1, generated_len])
1149 }
1150
1151 fn sample_next_token(
1158 logits: &[f32],
1159 temperature: f32,
1160 top_k: Option<usize>,
1161 top_p: Option<f32>,
1162 rng: &mut impl Rng,
1163 ) -> Result<u32> {
1164 if logits.is_empty() {
1165 return Err(tensor_op_error(
1166 "CommandRForCausalLM::sample_next_token",
1167 "logits must not be empty".to_string(),
1168 ));
1169 }
1170 if temperature <= 0.0 {
1171 return Ok(GenerationUtils::sample_greedy(logits));
1172 }
1173 if let Some(k) = top_k {
1174 let k = k.clamp(1, logits.len());
1175 return GenerationUtils::sample_top_k(logits, k, rng);
1176 }
1177 if let Some(p) = top_p {
1178 return GenerationUtils::sample_top_p(logits, p, rng);
1179 }
1180 let probs = GenerationUtils::softmax(logits);
1181 Ok(GenerationUtils::sample_from_probs(&probs, rng)? as u32)
1182 }
1183}
1184
1185#[derive(Debug, Clone)]
1187pub struct CommandRCausalLMOutput {
1188 pub loss: Option<Tensor>,
1189 pub logits: Tensor,
1190 pub past_key_values: Option<Vec<(Tensor, Tensor)>>,
1191 pub hidden_states: Option<Vec<Tensor>>,
1192 pub attentions: Option<Vec<Tensor>>,
1193}
1194
1195impl Model for CommandRForCausalLM {
1196 type Config = CommandRConfig;
1197 type Input = Tensor;
1198 type Output = Tensor;
1199
1200 fn forward(&self, input: Self::Input) -> Result<Self::Output> {
1201 let hidden_states = <CommandRModel as Model>::forward(&self.model, input)?;
1205
1206 let logits = self.lm_head.forward(hidden_states)?;
1208
1209 Ok(logits)
1210 }
1211
1212 fn load_pretrained(&mut self, reader: &mut dyn std::io::Read) -> Result<()> {
1218 let checkpoint = Checkpoint::from_reader(reader)?;
1219 self.model.load_checkpoint(&checkpoint, &["lm_head."])?;
1220
1221 let expected = [self.config.vocab_size, self.config.hidden_size];
1222 let head = match checkpoint.get("lm_head.weight") {
1223 Some(weight) => weight,
1224 None => {
1225 let embed_name = if checkpoint.contains("model.embed_tokens.weight") {
1226 "model.embed_tokens.weight"
1227 } else {
1228 "embed_tokens.weight"
1229 };
1230 checkpoint.get(embed_name).ok_or_else(|| {
1231 tensor_op_error(
1232 "CommandRForCausalLM::load_pretrained",
1233 "checkpoint holds neither lm_head.weight nor an embedding matrix to tie it to"
1234 .to_string(),
1235 )
1236 })?
1237 },
1238 };
1239 if head.shape() != expected {
1240 return Err(tensor_op_error(
1241 "CommandRForCausalLM::load_pretrained",
1242 format!(
1243 "language-model head has shape {:?} but this model expects {expected:?}",
1244 head.shape()
1245 ),
1246 ));
1247 }
1248 self.lm_head.set_weight(head.clone())?;
1249 Ok(())
1250 }
1251
1252 fn get_config(&self) -> &Self::Config {
1253 &self.config
1254 }
1255
1256 fn num_parameters(&self) -> usize {
1257 self.model.num_parameters() + self.lm_head.parameter_count()
1258 }
1259}
1260
1261impl CommandRForCausalLM {
1262 pub fn load_from_path(&mut self, model_path: impl AsRef<std::path::Path>) -> Result<()> {
1264 use crate::weight_loading::WeightLoadingConfig;
1265
1266 let config = WeightLoadingConfig {
1267 lazy_loading: true,
1268 memory_mapped: false,
1269 ..Default::default()
1270 };
1271 self.load_from_path_with_config(model_path, config)
1272 }
1273
1274 pub fn load_from_path_with_config(
1276 &mut self,
1277 model_path: impl AsRef<std::path::Path>,
1278 config: crate::weight_loading::WeightLoadingConfig,
1279 ) -> Result<()> {
1280 use crate::weight_loading::auto_create_loader;
1281
1282 let mut loader = auto_create_loader(model_path, Some(config))?;
1283
1284 if let Ok(embed_weights) = loader.load_tensor("model.embed_tokens.weight") {
1286 self.model.embed_tokens.set_weight(embed_weights)?;
1287 }
1288
1289 for (i, layer) in self.model.layers.iter_mut().enumerate() {
1291 let attn_prefix = format!("model.layers.{}.self_attn", i);
1293
1294 if let Ok(q_weight) = loader.load_tensor(&format!("{}.q_proj.weight", attn_prefix)) {
1295 layer.self_attn.q_proj.set_weight(q_weight)?;
1296 }
1297 if let Ok(k_weight) = loader.load_tensor(&format!("{}.k_proj.weight", attn_prefix)) {
1298 layer.self_attn.k_proj.set_weight(k_weight)?;
1299 }
1300 if let Ok(v_weight) = loader.load_tensor(&format!("{}.v_proj.weight", attn_prefix)) {
1301 layer.self_attn.v_proj.set_weight(v_weight)?;
1302 }
1303 if let Ok(o_weight) = loader.load_tensor(&format!("{}.o_proj.weight", attn_prefix)) {
1304 layer.self_attn.o_proj.set_weight(o_weight)?;
1305 }
1306
1307 let mlp_prefix = format!("model.layers.{}.mlp", i);
1309
1310 if let Ok(gate_weight) = loader.load_tensor(&format!("{}.gate_proj.weight", mlp_prefix))
1311 {
1312 layer.mlp.gate_proj.set_weight(gate_weight)?;
1313 }
1314 if let Ok(up_weight) = loader.load_tensor(&format!("{}.up_proj.weight", mlp_prefix)) {
1315 layer.mlp.up_proj.set_weight(up_weight)?;
1316 }
1317 if let Ok(down_weight) = loader.load_tensor(&format!("{}.down_proj.weight", mlp_prefix))
1318 {
1319 layer.mlp.down_proj.set_weight(down_weight)?;
1320 }
1321
1322 if let Ok(ln1_weight) =
1324 loader.load_tensor(&format!("model.layers.{}.input_layernorm.weight", i))
1325 {
1326 layer.input_layernorm.set_weight(ln1_weight)?;
1327 }
1328 if let Ok(ln2_weight) = loader.load_tensor(&format!(
1329 "model.layers.{}.post_attention_layernorm.weight",
1330 i
1331 )) {
1332 layer.post_attention_layernorm.set_weight(ln2_weight)?;
1333 }
1334 }
1335
1336 if let Ok(norm_weight) = loader.load_tensor("model.norm.weight") {
1338 self.model.norm.set_weight(norm_weight)?;
1339 }
1340
1341 if let Ok(lm_head_weight) = loader.load_tensor("lm_head.weight") {
1343 self.lm_head.set_weight(lm_head_weight)?;
1344 }
1345
1346 Ok(())
1347 }
1348
1349 pub fn load_from_huggingface(&mut self, model_name: &str) -> Result<()> {
1351 let cache_dir = std::env::var("HF_HOME")
1353 .or_else(|_| std::env::var("HUGGINGFACE_HUB_CACHE"))
1354 .unwrap_or_else(|_| {
1355 std::env::var("HOME").unwrap_or_else(|_| ".".to_string())
1356 + "/.cache/huggingface/hub"
1357 });
1358
1359 let model_path = std::path::Path::new(&cache_dir)
1360 .join(format!("models--{}", model_name.replace("/", "--")));
1361
1362 if model_path.exists() {
1363 self.load_from_path(&model_path)
1364 } else {
1365 self.download_from_huggingface_hub(model_name, &model_path)?;
1367 self.load_from_path(&model_path)
1368 }
1369 }
1370
1371 fn download_from_huggingface_hub(
1373 &self,
1374 model_name: &str,
1375 model_path: &std::path::Path,
1376 ) -> Result<()> {
1377 use std::process::Command;
1378
1379 tracing::info!(
1380 "Downloading model {} from HuggingFace Hub to {:?}",
1381 model_name,
1382 model_path
1383 );
1384
1385 std::fs::create_dir_all(model_path).map_err(|e| {
1387 trustformers_core::errors::TrustformersError::io_error(format!(
1388 "Failed to create model directory: {}",
1389 e
1390 ))
1391 })?;
1392
1393 let essential_files = vec![
1395 "config.json",
1396 "tokenizer.json",
1397 "tokenizer_config.json",
1398 "pytorch_model.bin", "model.safetensors", ];
1401
1402 let base_url = format!("https://huggingface.co/{}/resolve/main", model_name);
1403
1404 for file_name in &essential_files {
1406 let file_url = format!("{}/{}", base_url, file_name);
1407 let file_path = model_path.join(file_name);
1408
1409 tracing::info!("Attempting to download {}", file_url);
1410
1411 let file_path_str = file_path.to_str().ok_or_else(|| {
1413 TrustformersError::invalid_config(format!("Invalid UTF-8 in path: {:?}", file_path))
1414 })?;
1415
1416 let curl_result = Command::new("curl")
1418 .args([
1419 "-L", "-f", "-o",
1422 file_path_str,
1423 &file_url,
1424 ])
1425 .output();
1426
1427 match curl_result {
1428 Ok(output) if output.status.success() => {
1429 tracing::info!("Successfully downloaded {}", file_name);
1430 continue;
1431 },
1432 Ok(output) => {
1433 tracing::warn!(
1434 "Failed to download {} with curl: {}",
1435 file_name,
1436 String::from_utf8_lossy(&output.stderr)
1437 );
1438 },
1439 Err(e) => {
1440 tracing::info!("curl not available: {}", e);
1441 },
1442 }
1443
1444 let wget_result = Command::new("wget").args(["-O", file_path_str, &file_url]).output();
1446
1447 match wget_result {
1448 Ok(output) if output.status.success() => {
1449 tracing::info!("Successfully downloaded {} with wget", file_name);
1450 continue;
1451 },
1452 Ok(output) => {
1453 tracing::warn!(
1454 "Failed to download {} with wget: {}",
1455 file_name,
1456 String::from_utf8_lossy(&output.stderr)
1457 );
1458 },
1459 Err(e) => {
1460 tracing::info!("wget not available: {}", e);
1461 },
1462 }
1463
1464 if matches!(file_name, &"config.json" | &"pytorch_model.bin") {
1466 return Err(trustformers_core::errors::TrustformersError::io_error(format!(
1467 "Failed to download essential file {} for model {}. Please ensure curl or wget is installed and you have internet access.",
1468 file_name, model_name
1469 )));
1470 }
1471 }
1472
1473 tracing::info!(
1474 "Successfully downloaded model {} to {:?}",
1475 model_name,
1476 model_path
1477 );
1478 Ok(())
1479 }
1480
1481 pub fn load_with_mmap(&mut self, model_path: impl AsRef<std::path::Path>) -> Result<()> {
1490 use crate::weight_loading::WeightLoadingConfig;
1491
1492 let config = WeightLoadingConfig {
1493 lazy_loading: true,
1494 memory_mapped: true,
1495 streaming: false,
1496 ..Default::default()
1497 };
1498 self.load_from_path_with_config(model_path, config)
1499 }
1500
1501 #[deprecated(
1506 since = "0.2.1",
1507 note = "renamed to `load_with_mmap`: loading is memory-mapped, not deferred"
1508 )]
1509 pub fn load_with_lazy_loading(
1510 &mut self,
1511 model_path: impl AsRef<std::path::Path>,
1512 ) -> Result<()> {
1513 self.load_with_mmap(model_path)
1514 }
1515}
1516
1517impl Config for CommandRConfig {
1518 fn validate(&self) -> Result<()> {
1519 self.validate().map_err(|e| invalid_config("config_validation", &e))
1520 }
1521
1522 fn architecture(&self) -> &'static str {
1523 "command-r"
1524 }
1525}
1526
1527#[cfg(test)]
1528#[path = "model_tests.rs"]
1529mod tests;