trustformers_models/command_r/
tasks.rs1use thiserror::Error;
5
6#[derive(Debug, Error)]
8pub enum CommandRTaskError {
9 #[error("query must not be empty")]
11 EmptyQuery,
12 #[error("tool list must not be empty")]
14 NoTools,
15 #[error("document entry at index {0} has empty {1}")]
17 EmptyDocument(usize, &'static str),
18}
19
20pub const START_OF_TURN: &str = "<|START_OF_TURN_TOKEN|>";
24pub const END_OF_TURN: &str = "<|END_OF_TURN_TOKEN|>";
26pub const USER_TOKEN: &str = "<|USER_TOKEN|>";
28pub const CHATBOT_TOKEN: &str = "<|CHATBOT_TOKEN|>";
30pub const SYSTEM_TOKEN: &str = "<|SYSTEM_TOKEN|>";
32
33pub fn format_chat_prompt(
46 system: Option<&str>,
47 user_message: &str,
48) -> Result<String, CommandRTaskError> {
49 if user_message.trim().is_empty() {
50 return Err(CommandRTaskError::EmptyQuery);
51 }
52
53 let mut buf = String::new();
54
55 if let Some(sys) = system {
56 buf.push_str(START_OF_TURN);
57 buf.push_str(SYSTEM_TOKEN);
58 buf.push_str(sys);
59 buf.push_str(END_OF_TURN);
60 buf.push('\n');
61 }
62
63 buf.push_str(START_OF_TURN);
64 buf.push_str(USER_TOKEN);
65 buf.push_str(user_message);
66 buf.push_str(END_OF_TURN);
67 buf.push('\n');
68
69 buf.push_str(START_OF_TURN);
71 buf.push_str(CHATBOT_TOKEN);
72
73 Ok(buf)
74}
75
76#[derive(Debug, Clone)]
82pub struct RagDocument<'a> {
83 pub title: &'a str,
84 pub content: &'a str,
85}
86
87pub fn format_rag_prompt(
107 system_preamble: Option<&str>,
108 documents: &[RagDocument<'_>],
109 query: &str,
110) -> Result<String, CommandRTaskError> {
111 if query.trim().is_empty() {
112 return Err(CommandRTaskError::EmptyQuery);
113 }
114
115 for (idx, doc) in documents.iter().enumerate() {
116 if doc.title.trim().is_empty() {
117 return Err(CommandRTaskError::EmptyDocument(idx, "title"));
118 }
119 if doc.content.trim().is_empty() {
120 return Err(CommandRTaskError::EmptyDocument(idx, "content"));
121 }
122 }
123
124 let mut buf = String::new();
125
126 buf.push_str(START_OF_TURN);
128 buf.push_str(SYSTEM_TOKEN);
129
130 if let Some(preamble) = system_preamble {
131 buf.push_str(preamble);
132 buf.push('\n');
133 }
134
135 if documents.is_empty() {
136 buf.push_str("No supporting documents were retrieved. Answer based on your knowledge.");
137 } else {
138 buf.push_str(
139 "Use the documents below to answer the user's question. \
140 Cite the relevant document title(s) in your response.\n\n",
141 );
142 for (idx, doc) in documents.iter().enumerate() {
143 buf.push_str(&format!("Document [{}] (Title: {})\n", idx + 1, doc.title));
144 buf.push_str(doc.content);
145 buf.push_str("\n\n");
146 }
147 }
148
149 buf.push_str(END_OF_TURN);
150 buf.push('\n');
151
152 buf.push_str(START_OF_TURN);
154 buf.push_str(USER_TOKEN);
155 buf.push_str(query);
156 buf.push_str(END_OF_TURN);
157 buf.push('\n');
158
159 buf.push_str(START_OF_TURN);
161 buf.push_str(CHATBOT_TOKEN);
162
163 Ok(buf)
164}
165
166pub fn format_tool_use_prompt(tools: &[&str], query: &str) -> Result<String, CommandRTaskError> {
184 if tools.is_empty() {
185 return Err(CommandRTaskError::NoTools);
186 }
187 if query.trim().is_empty() {
188 return Err(CommandRTaskError::EmptyQuery);
189 }
190
191 let mut buf = String::new();
192
193 buf.push_str(START_OF_TURN);
195 buf.push_str(SYSTEM_TOKEN);
196 buf.push_str(
197 "You have access to the following tools. \
198 When appropriate, call a tool by emitting a JSON object \
199 with \"tool\" and \"parameters\" keys.\n\n",
200 );
201 buf.push_str("Available tools:\n");
202 for (idx, tool) in tools.iter().enumerate() {
203 buf.push_str(&format!(" {}. {}\n", idx + 1, tool));
204 }
205 buf.push_str(END_OF_TURN);
206 buf.push('\n');
207
208 buf.push_str(START_OF_TURN);
210 buf.push_str(USER_TOKEN);
211 buf.push_str(query);
212 buf.push_str(END_OF_TURN);
213 buf.push('\n');
214
215 buf.push_str(START_OF_TURN);
217 buf.push_str(CHATBOT_TOKEN);
218
219 Ok(buf)
220}
221
222pub fn apply_logit_scale(logits: &[f32], scale: f32) -> Vec<f32> {
239 logits.iter().map(|&x| x * scale).collect()
240}
241
242pub fn greedy_token(logits: &[f32]) -> Option<u32> {
247 logits
248 .iter()
249 .enumerate()
250 .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
251 .map(|(idx, _)| idx as u32)
252}
253
254#[cfg(test)]
257mod tests {
258 use super::*;
259
260 #[test]
263 fn test_command_r_config_default() {
264 use crate::command_r::CommandRConfig;
265 let cfg = CommandRConfig::default();
266 assert_eq!(cfg.vocab_size, 256000);
267 assert!(cfg.validate().is_ok());
268 }
269
270 #[test]
271 fn test_command_r_config_base() {
272 use crate::command_r::CommandRConfig;
273 let cfg = CommandRConfig::command_r();
274 assert_eq!(cfg.hidden_size, 8192);
275 assert_eq!(cfg.num_hidden_layers, 40);
276 assert_eq!(cfg.num_attention_heads, 64);
277 assert_eq!(cfg.num_key_value_heads, 64);
278 assert!(cfg.validate().is_ok());
279 }
280
281 #[test]
282 fn test_command_r_config_plus() {
283 use crate::command_r::CommandRConfig;
284 let cfg = CommandRConfig::command_r_plus();
285 assert_eq!(cfg.hidden_size, 12288);
286 assert_eq!(cfg.num_hidden_layers, 64);
287 assert_eq!(cfg.num_attention_heads, 96);
288 assert_eq!(cfg.num_key_value_heads, 96);
289 assert_eq!(cfg.intermediate_size, 33792);
290 assert!(cfg.validate().is_ok());
291 }
292
293 #[test]
296 fn test_command_r_layer_norm_basic() {
297 let x = vec![1.0_f32, 2.0, 3.0, 4.0];
299 let weight = vec![1.0_f32; 4];
300 let bias = vec![0.0_f32; 4];
301 let eps = 1e-5_f64;
302
303 let n = x.len() as f32;
304 let mean = x.iter().sum::<f32>() / n; let var = x.iter().map(|v| (v - mean).powi(2)).sum::<f32>() / n;
306 let std = (var + eps as f32).sqrt();
307 let expected: Vec<f32> = x
308 .iter()
309 .zip(weight.iter())
310 .zip(bias.iter())
311 .map(|((xi, wi), bi)| (xi - mean) / std * wi + bi)
312 .collect();
313
314 assert!((expected[0] - (-1.3416_f32)).abs() < 1e-3);
315 assert!((expected[3] - 1.3416_f32).abs() < 1e-3);
316 }
317
318 #[test]
319 fn test_command_r_layer_norm_zero_mean() {
320 let x = vec![1.0_f32, 3.0, 5.0, 7.0];
322 let n = x.len() as f32;
323 let mean = x.iter().sum::<f32>() / n;
324 let var = x.iter().map(|v| (v - mean).powi(2)).sum::<f32>() / n;
325 let std = (var + 1e-5_f32).sqrt();
326 let normed: Vec<f32> = x.iter().map(|xi| (xi - mean) / std).collect();
327 let normed_mean: f32 = normed.iter().sum::<f32>() / n;
328 assert!(
329 normed_mean.abs() < 1e-4,
330 "mean should be ~0, got {}",
331 normed_mean
332 );
333 }
334
335 #[test]
336 fn test_command_r_layernorm_vs_rmsnorm_differ() {
337 let x = vec![2.0_f32, 4.0, 6.0];
339 let n = x.len() as f32;
340 let eps = 1e-5_f32;
341
342 let mean_ln = x.iter().sum::<f32>() / n;
344 let var_ln = x.iter().map(|v| (v - mean_ln).powi(2)).sum::<f32>() / n;
345 let ln_out0 = (x[0] - mean_ln) / (var_ln + eps).sqrt();
346
347 let rms = (x.iter().map(|v| v * v).sum::<f32>() / n + eps).sqrt();
349 let rms_out0 = x[0] / rms;
350
351 assert!((ln_out0 - rms_out0).abs() > 1e-3);
353 }
354
355 #[test]
358 fn test_command_r_rope_standard() {
359 let dim = 8usize;
361 let base = 10000.0_f32;
362 let inv_freqs: Vec<f32> =
363 (0..dim / 2).map(|i| 1.0 / base.powf(2.0 * i as f32 / dim as f32)).collect();
364 assert!((inv_freqs[0] - 1.0_f32).abs() < 1e-6);
365 for i in 1..inv_freqs.len() {
366 assert!(inv_freqs[i] < inv_freqs[i - 1]);
367 }
368 }
369
370 #[test]
373 fn test_command_r_attention_gqa() {
374 use crate::command_r::config::CommandRConfig;
375 let cfg = CommandRConfig {
377 num_attention_heads: 32,
378 num_key_value_heads: 8,
379 hidden_size: 4096,
380 ..CommandRConfig::command_r()
381 };
382 assert_eq!(cfg.num_query_groups(), 4);
383 assert!(cfg.is_gqa());
384 }
385
386 #[test]
387 fn test_command_r_gqa_head_ratio() {
388 use crate::command_r::config::CommandRConfig;
389 let cfg = CommandRConfig {
391 num_attention_heads: 56,
392 num_key_value_heads: 8,
393 hidden_size: 7168,
394 ..CommandRConfig::command_r()
395 };
396 assert_eq!(cfg.num_query_groups(), 7);
397 }
398
399 #[test]
400 fn test_command_r_non_gqa_base() {
401 use crate::command_r::config::CommandRConfig;
402 let cfg = CommandRConfig::command_r();
404 assert!(!cfg.is_gqa());
405 assert_eq!(cfg.num_query_groups(), 1);
406 }
407
408 #[test]
411 fn test_command_r_mlp_swiglu() {
412 let x = 1.0_f32;
415 let sigmoid_x = 1.0 / (1.0 + (-x).exp());
416 let silu_x = x * sigmoid_x;
417 assert!((silu_x - 0.7311_f32).abs() < 1e-4);
418
419 let gate = vec![1.0_f32, -1.0, 0.5];
421 let up = vec![2.0_f32, 2.0, 2.0];
422 let swiglu: Vec<f32> = gate
423 .iter()
424 .zip(up.iter())
425 .map(|(&g, &u)| {
426 let s = 1.0 / (1.0 + (-g).exp());
427 g * s * u
428 })
429 .collect();
430 assert!(swiglu[0] > 0.0); assert!(swiglu[1] < 0.0); }
433
434 #[test]
437 fn test_command_r_logit_scale() {
438 let raw = vec![10.0_f32, 5.0, 0.0, -5.0];
439 let scaled = apply_logit_scale(&raw, 0.0625);
440 assert!((scaled[0] - 0.625_f32).abs() < 1e-6);
441 assert!((scaled[1] - 0.3125_f32).abs() < 1e-6);
442 assert!((scaled[2] - 0.0_f32).abs() < 1e-6);
443 assert!((scaled[3] - (-0.3125_f32)).abs() < 1e-6);
444 }
445
446 #[test]
447 fn test_command_r_model_forward() {
448 let logits = vec![0.1_f32, 0.9, 0.3, 0.7];
450 let token = greedy_token(&logits);
451 assert_eq!(token, Some(1u32)); }
453
454 #[test]
455 fn test_command_r_generate() {
456 let steps: Vec<Vec<f32>> = vec![
458 vec![0.1, 0.9, 0.2], vec![0.8, 0.1, 0.3], vec![0.2, 0.3, 0.7], ];
462 let generated: Vec<u32> = steps.iter().filter_map(|logits| greedy_token(logits)).collect();
463 assert_eq!(generated, vec![1u32, 0, 2]);
464 }
465
466 #[test]
469 fn test_command_r_rag_prompt_no_docs() {
470 let prompt = format_rag_prompt(None, &[], "What is the capital of France?")
471 .expect("should succeed with no docs");
472 assert!(prompt.contains(START_OF_TURN));
473 assert!(prompt.contains(SYSTEM_TOKEN));
474 assert!(prompt.contains("No supporting documents"));
475 assert!(prompt.contains("What is the capital of France?"));
476 assert!(prompt.ends_with(CHATBOT_TOKEN));
477 }
478
479 #[test]
480 fn test_command_r_rag_prompt_with_docs() {
481 let docs = vec![
482 RagDocument {
483 title: "Paris Guide",
484 content: "Paris is the capital of France.",
485 },
486 RagDocument {
487 title: "Europe Facts",
488 content: "France is a country in Western Europe.",
489 },
490 ];
491 let prompt = format_rag_prompt(None, &docs, "What is the capital of France?")
492 .expect("should succeed");
493 assert!(prompt.contains("Paris Guide"));
494 assert!(prompt.contains("Europe Facts"));
495 assert!(prompt.contains("Document [1]"));
496 assert!(prompt.contains("Document [2]"));
497 assert!(prompt.ends_with(CHATBOT_TOKEN));
498 }
499
500 #[test]
503 fn test_command_r_tool_use_prompt() {
504 let tools = &[
505 "search(query: str) -> List[str]",
506 "calculator(expr: str) -> float",
507 ];
508 let prompt = format_tool_use_prompt(tools, "What is 2 + 2?").expect("should succeed");
509 assert!(prompt.contains("search(query: str)"));
510 assert!(prompt.contains("calculator(expr: str)"));
511 assert!(prompt.contains("What is 2 + 2?"));
512 assert!(prompt.ends_with(CHATBOT_TOKEN));
513 }
514
515 #[test]
516 fn test_command_r_tool_use_prompt_empty_tools() {
517 let err = format_tool_use_prompt(&[], "hello");
518 assert!(matches!(err, Err(CommandRTaskError::NoTools)));
519 }
520
521 #[test]
524 fn test_command_r_chat_format() {
525 let prompt =
526 format_chat_prompt(Some("You are helpful."), "Hello!").expect("should succeed");
527 assert!(prompt.contains(SYSTEM_TOKEN));
528 assert!(prompt.contains("You are helpful."));
529 assert!(prompt.contains(USER_TOKEN));
530 assert!(prompt.contains("Hello!"));
531 assert!(prompt.ends_with(CHATBOT_TOKEN));
532 }
533
534 #[test]
535 fn test_command_r_chat_format_no_system() {
536 let prompt =
537 format_chat_prompt(None, "Hello!").expect("should succeed without system prompt");
538 assert!(!prompt.contains(SYSTEM_TOKEN));
539 assert!(prompt.contains(USER_TOKEN));
540 assert!(prompt.ends_with(CHATBOT_TOKEN));
541 }
542
543 #[test]
544 fn test_command_r_chat_format_empty_query() {
545 let err = format_chat_prompt(None, " ");
546 assert!(matches!(err, Err(CommandRTaskError::EmptyQuery)));
547 }
548
549 #[test]
552 fn test_command_r_error_display() {
553 let e1 = CommandRTaskError::EmptyQuery;
554 assert!(e1.to_string().contains("empty"));
555
556 let e2 = CommandRTaskError::NoTools;
557 assert!(e2.to_string().contains("empty"));
558
559 let e3 = CommandRTaskError::EmptyDocument(2, "title");
560 assert!(e3.to_string().contains("2"));
561 assert!(e3.to_string().contains("title"));
562 }
563}