vtcode_core/tools/summarizers/
mod.rs

1//! Tool result summarization strategies
2//!
3//! Implements Phase 4 summarization: Converting full tool outputs into
4//! concise LLM-friendly summaries while preserving rich UI content.
5//!
6//! ## Summarization Strategies
7//!
8//! Different tool types need different summarization approaches:
9//! - **Search tools** (grep, list): Count-based summaries
10//! - **File operations** (read, edit): Content previews and statistics
11//! - **Edit tools** (edit, patch): Diff statistics
12//! - **Execution tools** (bash, code): Output summaries
13
14use anyhow::Result;
15
16use crate::utils::tokens::{estimate_tokens, truncate_to_tokens};
17
18pub mod execution;
19pub mod file_ops;
20pub mod search;
21
22/// Token savings estimate from summarization.
23#[derive(Debug, Clone)]
24pub struct SavingsEstimate {
25    /// Token count of the summarized (LLM) output.
26    pub llm_tokens: usize,
27    /// Token count of the full (UI) output.
28    pub ui_tokens: usize,
29    /// Percentage of tokens saved (0.0 - 100.0).
30    pub savings_percent: f32,
31}
32
33use std::borrow::Cow;
34
35/// Truncate a line to max length with ellipsis (shared by execution + file_ops summarizers).
36///
37/// Returns `Cow::Borrowed` when no truncation is needed (zero allocation).
38pub(super) fn truncate_line<'a>(line: &'a str, max_len: usize) -> Cow<'a, str> {
39    if line.len() <= max_len {
40        Cow::Borrowed(line)
41    } else {
42        let prefix = vtcode_commons::formatting::truncate_utf8_prefix(line, max_len.saturating_sub(3));
43        Cow::Owned(format!("{prefix}..."))
44    }
45}
46
47/// Trait for tool result summarization strategies
48///
49/// Each tool type implements its own summarization logic
50/// to convert full output into concise LLM context
51pub trait Summarizer {
52    /// Summarize full output into concise LLM content
53    ///
54    /// # Arguments
55    /// * `full_output` - The complete tool output (for UI)
56    /// * `metadata` - Optional metadata about the operation
57    ///
58    /// # Returns
59    /// Concise summary optimized for LLM context (target: <100 tokens)
60    fn summarize(&self, full_output: &str, metadata: Option<&serde_json::Value>) -> Result<String>;
61
62    /// Estimate token savings from summarization
63    fn estimate_savings(&self, full_output: &str, summary: &str) -> SavingsEstimate {
64        let ui_tokens = estimate_tokens(full_output);
65        let llm_tokens = estimate_tokens(summary);
66        let savings = ui_tokens.saturating_sub(llm_tokens);
67        let savings_percent = if ui_tokens > 0 {
68            (savings as f32 / ui_tokens as f32) * 100.0
69        } else {
70            0.0
71        };
72        SavingsEstimate { llm_tokens, ui_tokens, savings_percent }
73    }
74}
75
76/// Extract key information from text (first N lines, keywords, etc.)
77///
78/// Useful for command output, file content, etc.
79pub fn extract_key_info(text: &str, max_lines: usize) -> String {
80    let mut lines: Vec<&str> = Vec::with_capacity(max_lines.min(32));
81    let mut total_lines = 0usize;
82    for line in text.lines() {
83        total_lines += 1;
84        if lines.len() < max_lines {
85            lines.push(line);
86        }
87    }
88
89    if total_lines > max_lines {
90        format!("{}\n[...{} more lines]", lines.join("\n"), total_lines - max_lines)
91    } else {
92        lines.join("\n")
93    }
94}
95
96#[cfg(test)]
97mod tests {
98    use super::*;
99
100    #[test]
101    fn test_estimate_tokens() {
102        // tiktoken cl100k_base BPE tokenizes "Hello world" as 2 tokens
103        assert_eq!(estimate_tokens("Hello world"), 2);
104        assert_eq!(estimate_tokens(""), 0);
105        // Repeated single chars are highly compressible: ~8 chars/token for 'a'
106        assert_eq!(estimate_tokens("a".repeat(1000).as_str()), 125);
107    }
108
109    #[test]
110    fn test_truncate_to_tokens() {
111        let text = "a".repeat(1000);
112        let truncated = truncate_to_tokens(&text, 50); // 50 tokens ≈ 400 chars at 8 chars/token
113        // BPE with repeated chars is more efficient (~8 chars/token)
114        assert!(truncated.len() <= 500); // generous bound for BPE variance
115        // BPE decode of repeated chars succeeds without "..." fallback
116        assert!(truncated.len() < text.len());
117    }
118
119    #[test]
120    fn test_truncate_short_text() {
121        let text = "Short text";
122        let truncated = truncate_to_tokens(text, 100);
123        assert_eq!(truncated, text);
124    }
125
126    #[test]
127    fn test_extract_key_info() {
128        let text = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5";
129        let extracted = extract_key_info(text, 3);
130        assert!(extracted.contains("Line 1"));
131        assert!(extracted.contains("Line 3"));
132        assert!(extracted.contains("[...2 more lines]"));
133    }
134
135    #[test]
136    fn test_extract_key_info_exact() {
137        let text = "Line 1\nLine 2\nLine 3";
138        let extracted = extract_key_info(text, 3);
139        assert_eq!(extracted, text);
140        assert!(!extracted.contains("more lines"));
141    }
142}
vtcode_core/tools/summarizers/mod.rs

vtcode_core/tools/summarizers/
mod.rs
