Skip to main content

trustformers_models/command_r/
tasks.rs

1//! Command-R task-specific utilities: RAG prompt formatting, tool use formatting,
2//! and extended CausalLM interfaces for Cohere Command-R models.
3
4use thiserror::Error;
5
6/// Errors specific to Command-R task operations.
7#[derive(Debug, Error)]
8pub enum CommandRTaskError {
9    /// An empty query was provided.
10    #[error("query must not be empty")]
11    EmptyQuery,
12    /// Tool list was empty when at least one tool is required.
13    #[error("tool list must not be empty")]
14    NoTools,
15    /// A document title or snippet was empty.
16    #[error("document entry at index {0} has empty {1}")]
17    EmptyDocument(usize, &'static str),
18}
19
20// ─── Special token constants ───────────────────────────────────────────────
21
22/// Start-of-turn delimiter used in Command-R's chat format.
23pub const START_OF_TURN: &str = "<|START_OF_TURN_TOKEN|>";
24/// End-of-turn delimiter used in Command-R's chat format.
25pub const END_OF_TURN: &str = "<|END_OF_TURN_TOKEN|>";
26/// Role token for the user turn.
27pub const USER_TOKEN: &str = "<|USER_TOKEN|>";
28/// Role token for the assistant (chatbot) turn.
29pub const CHATBOT_TOKEN: &str = "<|CHATBOT_TOKEN|>";
30/// Role token for a system-level instruction.
31pub const SYSTEM_TOKEN: &str = "<|SYSTEM_TOKEN|>";
32
33// ─── Chat format ───────────────────────────────────────────────────────────
34
35/// Format a simple user message for Command-R's chat format.
36///
37/// The resulting string is ready to be tokenised and fed to the model.
38/// The model should then generate the assistant reply starting from
39/// the open `<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>` sentinel that is
40/// appended at the end.
41///
42/// # Errors
43///
44/// Returns [`CommandRTaskError::EmptyQuery`] when `user_message` is empty.
45pub fn format_chat_prompt(
46    system: Option<&str>,
47    user_message: &str,
48) -> Result<String, CommandRTaskError> {
49    if user_message.trim().is_empty() {
50        return Err(CommandRTaskError::EmptyQuery);
51    }
52
53    let mut buf = String::new();
54
55    if let Some(sys) = system {
56        buf.push_str(START_OF_TURN);
57        buf.push_str(SYSTEM_TOKEN);
58        buf.push_str(sys);
59        buf.push_str(END_OF_TURN);
60        buf.push('\n');
61    }
62
63    buf.push_str(START_OF_TURN);
64    buf.push_str(USER_TOKEN);
65    buf.push_str(user_message);
66    buf.push_str(END_OF_TURN);
67    buf.push('\n');
68
69    // Open the assistant turn; the model generates text from this point.
70    buf.push_str(START_OF_TURN);
71    buf.push_str(CHATBOT_TOKEN);
72
73    Ok(buf)
74}
75
76// ─── RAG prompt ────────────────────────────────────────────────────────────
77
78/// A document passed to the RAG prompt formatter.
79///
80/// `title` is shown as a heading, `content` is the passage body.
81#[derive(Debug, Clone)]
82pub struct RagDocument<'a> {
83    pub title: &'a str,
84    pub content: &'a str,
85}
86
87/// Format a Retrieval-Augmented Generation (RAG) prompt for Command-R.
88///
89/// Constructs a system prompt that lists the supplied grounding documents and
90/// instructs the model to answer only from those documents, followed by a user
91/// turn containing the query.
92///
93/// # Arguments
94///
95/// * `system_preamble` – Optional extra instruction prepended to the system turn
96///   (e.g. persona or safety guidance).
97/// * `documents` – Slice of `(title, content)` pairs representing retrieved passages.
98///   Accepts an empty slice; in that case the system turn simply instructs the
99///   model that no supporting documents were retrieved.
100/// * `query` – The user's question.
101///
102/// # Errors
103///
104/// * [`CommandRTaskError::EmptyQuery`] if `query` is empty.
105/// * [`CommandRTaskError::EmptyDocument`] if any document has an empty title or content.
106pub fn format_rag_prompt(
107    system_preamble: Option<&str>,
108    documents: &[RagDocument<'_>],
109    query: &str,
110) -> Result<String, CommandRTaskError> {
111    if query.trim().is_empty() {
112        return Err(CommandRTaskError::EmptyQuery);
113    }
114
115    for (idx, doc) in documents.iter().enumerate() {
116        if doc.title.trim().is_empty() {
117            return Err(CommandRTaskError::EmptyDocument(idx, "title"));
118        }
119        if doc.content.trim().is_empty() {
120            return Err(CommandRTaskError::EmptyDocument(idx, "content"));
121        }
122    }
123
124    let mut buf = String::new();
125
126    // System turn
127    buf.push_str(START_OF_TURN);
128    buf.push_str(SYSTEM_TOKEN);
129
130    if let Some(preamble) = system_preamble {
131        buf.push_str(preamble);
132        buf.push('\n');
133    }
134
135    if documents.is_empty() {
136        buf.push_str("No supporting documents were retrieved. Answer based on your knowledge.");
137    } else {
138        buf.push_str(
139            "Use the documents below to answer the user's question. \
140             Cite the relevant document title(s) in your response.\n\n",
141        );
142        for (idx, doc) in documents.iter().enumerate() {
143            buf.push_str(&format!("Document [{}] (Title: {})\n", idx + 1, doc.title));
144            buf.push_str(doc.content);
145            buf.push_str("\n\n");
146        }
147    }
148
149    buf.push_str(END_OF_TURN);
150    buf.push('\n');
151
152    // User turn
153    buf.push_str(START_OF_TURN);
154    buf.push_str(USER_TOKEN);
155    buf.push_str(query);
156    buf.push_str(END_OF_TURN);
157    buf.push('\n');
158
159    // Open assistant turn
160    buf.push_str(START_OF_TURN);
161    buf.push_str(CHATBOT_TOKEN);
162
163    Ok(buf)
164}
165
166// ─── Tool-use prompt ───────────────────────────────────────────────────────
167
168/// Format a tool-use prompt for Command-R.
169///
170/// The model will be shown the available tool signatures and then asked the
171/// user's query so it can decide which (if any) tool to call.
172///
173/// # Arguments
174///
175/// * `tools` – Slice of tool descriptions (e.g. `"search(query: str) -> str"`).
176///   Must not be empty.
177/// * `query` – The user's request.
178///
179/// # Errors
180///
181/// * [`CommandRTaskError::NoTools`] if the slice is empty.
182/// * [`CommandRTaskError::EmptyQuery`] if `query` is empty.
183pub fn format_tool_use_prompt(tools: &[&str], query: &str) -> Result<String, CommandRTaskError> {
184    if tools.is_empty() {
185        return Err(CommandRTaskError::NoTools);
186    }
187    if query.trim().is_empty() {
188        return Err(CommandRTaskError::EmptyQuery);
189    }
190
191    let mut buf = String::new();
192
193    // System turn: list available tools
194    buf.push_str(START_OF_TURN);
195    buf.push_str(SYSTEM_TOKEN);
196    buf.push_str(
197        "You have access to the following tools. \
198         When appropriate, call a tool by emitting a JSON object \
199         with \"tool\" and \"parameters\" keys.\n\n",
200    );
201    buf.push_str("Available tools:\n");
202    for (idx, tool) in tools.iter().enumerate() {
203        buf.push_str(&format!("  {}. {}\n", idx + 1, tool));
204    }
205    buf.push_str(END_OF_TURN);
206    buf.push('\n');
207
208    // User turn
209    buf.push_str(START_OF_TURN);
210    buf.push_str(USER_TOKEN);
211    buf.push_str(query);
212    buf.push_str(END_OF_TURN);
213    buf.push('\n');
214
215    // Open assistant turn
216    buf.push_str(START_OF_TURN);
217    buf.push_str(CHATBOT_TOKEN);
218
219    Ok(buf)
220}
221
222// ─── Logit-scale helpers ───────────────────────────────────────────────────
223
224/// Apply a scalar logit-scale factor to raw model logits.
225///
226/// Command-R multiplies final logits by `logit_scale` (default 0.0625 = 1/16)
227/// to keep them in a numerically stable range for the softmax.
228///
229/// # Arguments
230///
231/// * `logits` – A flat `[seq_len * vocab_size]` or `[vocab_size]` slice produced
232///   by the LM head.
233/// * `scale` – The `logit_scale` value from [`super::CommandRConfig`].
234///
235/// # Returns
236///
237/// A new `Vec<f32>` with each element multiplied by `scale`.
238pub fn apply_logit_scale(logits: &[f32], scale: f32) -> Vec<f32> {
239    logits.iter().map(|&x| x * scale).collect()
240}
241
242/// Greedy-decode one token from a logit slice of shape `[vocab_size]`.
243///
244/// Returns the index of the maximum logit.  This is a deterministic, zero-copy
245/// scan with no `unwrap`.
246pub fn greedy_token(logits: &[f32]) -> Option<u32> {
247    logits
248        .iter()
249        .enumerate()
250        .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
251        .map(|(idx, _)| idx as u32)
252}
253
254// ─── Tests ─────────────────────────────────────────────────────────────────
255
256#[cfg(test)]
257mod tests {
258    use super::*;
259
260    // ── Config tests ──────────────────────────────────────────────────────
261
262    #[test]
263    fn test_command_r_config_default() {
264        use crate::command_r::CommandRConfig;
265        let cfg = CommandRConfig::default();
266        assert_eq!(cfg.vocab_size, 256000);
267        assert!(cfg.validate().is_ok());
268    }
269
270    #[test]
271    fn test_command_r_config_base() {
272        use crate::command_r::CommandRConfig;
273        let cfg = CommandRConfig::command_r();
274        assert_eq!(cfg.hidden_size, 8192);
275        assert_eq!(cfg.num_hidden_layers, 40);
276        assert_eq!(cfg.num_attention_heads, 64);
277        assert_eq!(cfg.num_key_value_heads, 64);
278        assert!(cfg.validate().is_ok());
279    }
280
281    #[test]
282    fn test_command_r_config_plus() {
283        use crate::command_r::CommandRConfig;
284        let cfg = CommandRConfig::command_r_plus();
285        assert_eq!(cfg.hidden_size, 12288);
286        assert_eq!(cfg.num_hidden_layers, 64);
287        assert_eq!(cfg.num_attention_heads, 96);
288        assert_eq!(cfg.num_key_value_heads, 96);
289        assert_eq!(cfg.intermediate_size, 33792);
290        assert!(cfg.validate().is_ok());
291    }
292
293    // ── LayerNorm tests ───────────────────────────────────────────────────
294
295    #[test]
296    fn test_command_r_layer_norm_basic() {
297        // LayerNorm: mean-subtraction + divide by std
298        let x = vec![1.0_f32, 2.0, 3.0, 4.0];
299        let weight = vec![1.0_f32; 4];
300        let bias = vec![0.0_f32; 4];
301        let eps = 1e-5_f64;
302
303        let n = x.len() as f32;
304        let mean = x.iter().sum::<f32>() / n; // 2.5
305        let var = x.iter().map(|v| (v - mean).powi(2)).sum::<f32>() / n;
306        let std = (var + eps as f32).sqrt();
307        let expected: Vec<f32> = x
308            .iter()
309            .zip(weight.iter())
310            .zip(bias.iter())
311            .map(|((xi, wi), bi)| (xi - mean) / std * wi + bi)
312            .collect();
313
314        assert!((expected[0] - (-1.3416_f32)).abs() < 1e-3);
315        assert!((expected[3] - 1.3416_f32).abs() < 1e-3);
316    }
317
318    #[test]
319    fn test_command_r_layer_norm_zero_mean() {
320        // After LayerNorm the output should have near-zero mean (with unit weights, zero bias).
321        let x = vec![1.0_f32, 3.0, 5.0, 7.0];
322        let n = x.len() as f32;
323        let mean = x.iter().sum::<f32>() / n;
324        let var = x.iter().map(|v| (v - mean).powi(2)).sum::<f32>() / n;
325        let std = (var + 1e-5_f32).sqrt();
326        let normed: Vec<f32> = x.iter().map(|xi| (xi - mean) / std).collect();
327        let normed_mean: f32 = normed.iter().sum::<f32>() / n;
328        assert!(
329            normed_mean.abs() < 1e-4,
330            "mean should be ~0, got {}",
331            normed_mean
332        );
333    }
334
335    #[test]
336    fn test_command_r_layernorm_vs_rmsnorm_differ() {
337        // RMSNorm does NOT subtract the mean; LayerNorm does.
338        let x = vec![2.0_f32, 4.0, 6.0];
339        let n = x.len() as f32;
340        let eps = 1e-5_f32;
341
342        // LayerNorm
343        let mean_ln = x.iter().sum::<f32>() / n;
344        let var_ln = x.iter().map(|v| (v - mean_ln).powi(2)).sum::<f32>() / n;
345        let ln_out0 = (x[0] - mean_ln) / (var_ln + eps).sqrt();
346
347        // RMSNorm
348        let rms = (x.iter().map(|v| v * v).sum::<f32>() / n + eps).sqrt();
349        let rms_out0 = x[0] / rms;
350
351        // They should produce different values for a non-zero-mean input.
352        assert!((ln_out0 - rms_out0).abs() > 1e-3);
353    }
354
355    // ── RoPE test ─────────────────────────────────────────────────────────
356
357    #[test]
358    fn test_command_r_rope_standard() {
359        // Verify that freq[0] = 1.0 and freq decreases with increasing dimension index.
360        let dim = 8usize;
361        let base = 10000.0_f32;
362        let inv_freqs: Vec<f32> =
363            (0..dim / 2).map(|i| 1.0 / base.powf(2.0 * i as f32 / dim as f32)).collect();
364        assert!((inv_freqs[0] - 1.0_f32).abs() < 1e-6);
365        for i in 1..inv_freqs.len() {
366            assert!(inv_freqs[i] < inv_freqs[i - 1]);
367        }
368    }
369
370    // ── Attention / GQA tests ─────────────────────────────────────────────
371
372    #[test]
373    fn test_command_r_attention_gqa() {
374        use crate::command_r::config::CommandRConfig;
375        // Build a custom GQA config: 32 query heads, 8 KV heads → 4 groups
376        let cfg = CommandRConfig {
377            num_attention_heads: 32,
378            num_key_value_heads: 8,
379            hidden_size: 4096,
380            ..CommandRConfig::command_r()
381        };
382        assert_eq!(cfg.num_query_groups(), 4);
383        assert!(cfg.is_gqa());
384    }
385
386    #[test]
387    fn test_command_r_gqa_head_ratio() {
388        use crate::command_r::config::CommandRConfig;
389        // 56 query heads, 8 KV heads → 7 queries per KV head
390        let cfg = CommandRConfig {
391            num_attention_heads: 56,
392            num_key_value_heads: 8,
393            hidden_size: 7168,
394            ..CommandRConfig::command_r()
395        };
396        assert_eq!(cfg.num_query_groups(), 7);
397    }
398
399    #[test]
400    fn test_command_r_non_gqa_base() {
401        use crate::command_r::config::CommandRConfig;
402        // Base command_r has equal query and KV heads (non-GQA)
403        let cfg = CommandRConfig::command_r();
404        assert!(!cfg.is_gqa());
405        assert_eq!(cfg.num_query_groups(), 1);
406    }
407
408    // ── MLP / SwiGLU tests ────────────────────────────────────────────────
409
410    #[test]
411    fn test_command_r_mlp_swiglu() {
412        // SwiGLU: output = gate * silu(gate) * up (element-wise)
413        // Here we verify the SiLU component: silu(x) = x * sigmoid(x)
414        let x = 1.0_f32;
415        let sigmoid_x = 1.0 / (1.0 + (-x).exp());
416        let silu_x = x * sigmoid_x;
417        assert!((silu_x - 0.7311_f32).abs() < 1e-4);
418
419        // SwiGLU for a small vector
420        let gate = vec![1.0_f32, -1.0, 0.5];
421        let up = vec![2.0_f32, 2.0, 2.0];
422        let swiglu: Vec<f32> = gate
423            .iter()
424            .zip(up.iter())
425            .map(|(&g, &u)| {
426                let s = 1.0 / (1.0 + (-g).exp());
427                g * s * u
428            })
429            .collect();
430        assert!(swiglu[0] > 0.0); // positive input → positive output
431        assert!(swiglu[1] < 0.0); // negative input → negative output
432    }
433
434    // ── Model forward / logit-scale tests ─────────────────────────────────
435
436    #[test]
437    fn test_command_r_logit_scale() {
438        let raw = vec![10.0_f32, 5.0, 0.0, -5.0];
439        let scaled = apply_logit_scale(&raw, 0.0625);
440        assert!((scaled[0] - 0.625_f32).abs() < 1e-6);
441        assert!((scaled[1] - 0.3125_f32).abs() < 1e-6);
442        assert!((scaled[2] - 0.0_f32).abs() < 1e-6);
443        assert!((scaled[3] - (-0.3125_f32)).abs() < 1e-6);
444    }
445
446    #[test]
447    fn test_command_r_model_forward() {
448        // Minimal smoke-test: greedy decode on a tiny logit vector.
449        let logits = vec![0.1_f32, 0.9, 0.3, 0.7];
450        let token = greedy_token(&logits);
451        assert_eq!(token, Some(1u32)); // index 1 has the highest value
452    }
453
454    #[test]
455    fn test_command_r_generate() {
456        // Greedy generation from a sequence of per-step logit sets.
457        let steps: Vec<Vec<f32>> = vec![
458            vec![0.1, 0.9, 0.2], // step 0 → token 1
459            vec![0.8, 0.1, 0.3], // step 1 → token 0
460            vec![0.2, 0.3, 0.7], // step 2 → token 2
461        ];
462        let generated: Vec<u32> = steps.iter().filter_map(|logits| greedy_token(logits)).collect();
463        assert_eq!(generated, vec![1u32, 0, 2]);
464    }
465
466    // ── RAG prompt tests ──────────────────────────────────────────────────
467
468    #[test]
469    fn test_command_r_rag_prompt_no_docs() {
470        let prompt = format_rag_prompt(None, &[], "What is the capital of France?")
471            .expect("should succeed with no docs");
472        assert!(prompt.contains(START_OF_TURN));
473        assert!(prompt.contains(SYSTEM_TOKEN));
474        assert!(prompt.contains("No supporting documents"));
475        assert!(prompt.contains("What is the capital of France?"));
476        assert!(prompt.ends_with(CHATBOT_TOKEN));
477    }
478
479    #[test]
480    fn test_command_r_rag_prompt_with_docs() {
481        let docs = vec![
482            RagDocument {
483                title: "Paris Guide",
484                content: "Paris is the capital of France.",
485            },
486            RagDocument {
487                title: "Europe Facts",
488                content: "France is a country in Western Europe.",
489            },
490        ];
491        let prompt = format_rag_prompt(None, &docs, "What is the capital of France?")
492            .expect("should succeed");
493        assert!(prompt.contains("Paris Guide"));
494        assert!(prompt.contains("Europe Facts"));
495        assert!(prompt.contains("Document [1]"));
496        assert!(prompt.contains("Document [2]"));
497        assert!(prompt.ends_with(CHATBOT_TOKEN));
498    }
499
500    // ── Tool-use prompt tests ─────────────────────────────────────────────
501
502    #[test]
503    fn test_command_r_tool_use_prompt() {
504        let tools = &[
505            "search(query: str) -> List[str]",
506            "calculator(expr: str) -> float",
507        ];
508        let prompt = format_tool_use_prompt(tools, "What is 2 + 2?").expect("should succeed");
509        assert!(prompt.contains("search(query: str)"));
510        assert!(prompt.contains("calculator(expr: str)"));
511        assert!(prompt.contains("What is 2 + 2?"));
512        assert!(prompt.ends_with(CHATBOT_TOKEN));
513    }
514
515    #[test]
516    fn test_command_r_tool_use_prompt_empty_tools() {
517        let err = format_tool_use_prompt(&[], "hello");
518        assert!(matches!(err, Err(CommandRTaskError::NoTools)));
519    }
520
521    // ── Chat format test ──────────────────────────────────────────────────
522
523    #[test]
524    fn test_command_r_chat_format() {
525        let prompt =
526            format_chat_prompt(Some("You are helpful."), "Hello!").expect("should succeed");
527        assert!(prompt.contains(SYSTEM_TOKEN));
528        assert!(prompt.contains("You are helpful."));
529        assert!(prompt.contains(USER_TOKEN));
530        assert!(prompt.contains("Hello!"));
531        assert!(prompt.ends_with(CHATBOT_TOKEN));
532    }
533
534    #[test]
535    fn test_command_r_chat_format_no_system() {
536        let prompt =
537            format_chat_prompt(None, "Hello!").expect("should succeed without system prompt");
538        assert!(!prompt.contains(SYSTEM_TOKEN));
539        assert!(prompt.contains(USER_TOKEN));
540        assert!(prompt.ends_with(CHATBOT_TOKEN));
541    }
542
543    #[test]
544    fn test_command_r_chat_format_empty_query() {
545        let err = format_chat_prompt(None, "  ");
546        assert!(matches!(err, Err(CommandRTaskError::EmptyQuery)));
547    }
548
549    // ── Error display tests ───────────────────────────────────────────────
550
551    #[test]
552    fn test_command_r_error_display() {
553        let e1 = CommandRTaskError::EmptyQuery;
554        assert!(e1.to_string().contains("empty"));
555
556        let e2 = CommandRTaskError::NoTools;
557        assert!(e2.to_string().contains("empty"));
558
559        let e3 = CommandRTaskError::EmptyDocument(2, "title");
560        assert!(e3.to_string().contains("2"));
561        assert!(e3.to_string().contains("title"));
562    }
563}