vtcode_core/tools/
result_metadata.rs

1//! Tool result metadata and quality scoring
2//!
3//! Provides metadata about tool result quality, confidence, and usefulness.
4//! This allows the agent to make informed decisions about result reliability
5//! and prioritize high-quality results in context windows.
6//!
7//! Core types (`ResultCompleteness`, `ResultMetadata`, `EnhancedToolResult`, `ResultScorer`)
8//! are defined in `vtcode-commons::tool_types` and re-exported here for backward compatibility.
9//! This module retains the scorer implementations that depend on tool_intent.
10
11// Re-export core types from vtcode-commons::tool_types for backward compatibility
12pub use vtcode_commons::tool_types::{
13    CompactStr, EnhancedToolResult, ResultCompleteness, ResultMetadata, ResultScorer,
14};
15
16use crate::config::constants::tools;
17use crate::tools::tool_intent;
18use hashbrown::HashMap;
19use serde_json::Value;
20
21/// Scorer for bounded code search results.
22pub struct CodeSearchScorer;
23
24impl ResultScorer for CodeSearchScorer {
25    fn score(&self, result: &Value) -> ResultMetadata {
26        let mut metadata = ResultMetadata::default();
27        metadata.content_types.push("code".to_string());
28
29        match result {
30            Value::Object(map) => {
31                if let Some(results) = map.get("results")
32                    && let Some(entries) = results.as_array()
33                {
34                    metadata.result_count = entries.len();
35
36                    metadata.confidence = if entries.len() > 5 {
37                        0.85
38                    } else if !entries.is_empty() {
39                        0.80
40                    } else {
41                        1.0
42                    };
43
44                    metadata.relevance = 0.75;
45                    metadata.completeness = if map.get("truncated").and_then(Value::as_bool).unwrap_or(false) {
46                        ResultCompleteness::Partial
47                    } else {
48                        ResultCompleteness::Complete
49                    };
50                    metadata.false_positive_likelihood = 0.05;
51                }
52            }
53            Value::Array(arr) => {
54                metadata.result_count = arr.len();
55                metadata.confidence = if arr.is_empty() { 1.0 } else { 0.80 };
56                metadata.relevance = 0.75;
57            }
58            _ => {
59                metadata = ResultMetadata::error();
60            }
61        }
62
63        metadata
64    }
65
66    fn tool_name(&self) -> &str {
67        tools::CODE_SEARCH
68    }
69}
70
71/// Scorer for file finding results
72pub struct FindScorer;
73
74impl ResultScorer for FindScorer {
75    fn score(&self, result: &Value) -> ResultMetadata {
76        let mut metadata = ResultMetadata::default();
77        metadata.content_types.push("filesystem".to_string());
78
79        match result {
80            Value::Object(map) => {
81                if let Some(files) = map.get("files")
82                    && let Some(file_arr) = files.as_array()
83                {
84                    metadata.result_count = file_arr.len();
85                    metadata.confidence = if file_arr.is_empty() {
86                        1.0 // High confidence in "no files"
87                    } else {
88                        0.90 // Very high confidence in file paths
89                    };
90                    metadata.relevance = 0.80;
91                    metadata.completeness = ResultCompleteness::Complete;
92                }
93            }
94            Value::Array(arr) => {
95                metadata.result_count = arr.len();
96                metadata.confidence = 0.90;
97                metadata.relevance = 0.80;
98            }
99            _ => {
100                metadata = ResultMetadata::error();
101            }
102        }
103
104        metadata
105    }
106
107    fn tool_name(&self) -> &str {
108        "find"
109    }
110}
111
112/// Scorer for shell command results
113pub struct ShellScorer;
114
115impl ResultScorer for ShellScorer {
116    fn score(&self, result: &Value) -> ResultMetadata {
117        let mut metadata = ResultMetadata::default();
118
119        match result {
120            Value::Object(map) => {
121                // Check for exit code
122                let exit_code = map.get("exit_code").and_then(|v| v.as_i64()).unwrap_or(-1);
123
124                // Success means high confidence
125                if exit_code == 0 {
126                    metadata.confidence = 0.85;
127                } else {
128                    metadata.confidence = 0.20;
129                    metadata.completeness = ResultCompleteness::Empty;
130                }
131
132                if let Some(output) = map.get("output").or_else(|| map.get("stdout"))
133                    && let Some(s) = output.as_str()
134                {
135                    metadata.result_count = s.lines().count();
136                    metadata.relevance = 0.70;
137                }
138            }
139            _ => {
140                metadata = ResultMetadata::error();
141            }
142        }
143
144        metadata
145    }
146
147    fn tool_name(&self) -> &str {
148        tools::UNIFIED_EXEC
149    }
150}
151
152/// Registry for result scorers
153pub struct ScorerRegistry {
154    scorers: HashMap<CompactStr, Box<dyn ResultScorer>>,
155}
156
157impl ScorerRegistry {
158    pub fn new() -> Self {
159        let mut scorers: HashMap<CompactStr, Box<dyn ResultScorer>> = HashMap::new();
160        scorers.insert(CompactStr::from(tools::CODE_SEARCH), Box::new(CodeSearchScorer) as Box<dyn ResultScorer>);
161        scorers.insert(CompactStr::from("find"), Box::new(FindScorer) as Box<dyn ResultScorer>);
162        scorers.insert(CompactStr::from(tools::UNIFIED_EXEC), Box::new(ShellScorer) as Box<dyn ResultScorer>);
163
164        Self { scorers }
165    }
166
167    /// Register a custom scorer
168    pub fn register(&mut self, scorer: Box<dyn ResultScorer>) {
169        self.scorers.insert(CompactStr::from(scorer.tool_name()), scorer);
170    }
171
172    /// Score a tool result
173    pub fn score(&self, tool_name: &str, result: &Value) -> ResultMetadata {
174        let canonical_tool_name = tool_intent::canonical_command_session_tool_name(tool_name).unwrap_or(tool_name);
175        if let Some(scorer) = self.scorers.get(canonical_tool_name) {
176            scorer.score(result)
177        } else {
178            // Default scoring for unknown tools
179            match result {
180                Value::Null => ResultMetadata::empty(),
181                Value::Object(_) => ResultMetadata::success(0.6, 0.6),
182                Value::Array(arr) => {
183                    let mut meta = ResultMetadata::success(0.6, 0.6);
184                    meta.result_count = arr.len();
185                    meta
186                }
187                _ => ResultMetadata::success(0.5, 0.5),
188            }
189        }
190    }
191}
192
193impl Default for ScorerRegistry {
194    fn default() -> Self {
195        Self::new()
196    }
197}
198
199#[cfg(test)]
200mod tests {
201    use super::*;
202    use serde_json::json;
203
204    #[test]
205    fn test_result_completeness() {
206        assert_eq!(ResultCompleteness::Complete.to_string(), "complete");
207        assert_eq!(ResultCompleteness::Partial.to_string(), "partial");
208        assert_eq!(ResultCompleteness::Empty.to_string(), "empty");
209    }
210
211    #[test]
212    fn test_quality_score() {
213        let meta = ResultMetadata {
214            confidence: 0.8,
215            relevance: 0.8,
216            false_positive_likelihood: 0.1,
217            ..Default::default()
218        };
219
220        let score = meta.quality_score();
221        assert!(score > 0.6 && score < 0.8);
222    }
223
224    #[test]
225    fn test_enhanced_result_is_useful() {
226        let result = EnhancedToolResult::new(
227            json!({"results": [], "returned": 0, "truncated": false}),
228            ResultMetadata::success(0.8, 0.8),
229            tools::CODE_SEARCH.to_string(),
230        );
231
232        assert!(result.is_useful());
233        assert!(!result.is_high_quality());
234    }
235
236    #[test]
237    fn test_code_search_scorer() {
238        let scorer = CodeSearchScorer;
239        let result = json!({
240            "results": ["line1", "line2", "line3"],
241            "returned": 3,
242            "truncated": false
243        });
244
245        let meta = scorer.score(&result);
246        assert_eq!(meta.result_count, 3);
247        assert!(meta.confidence > 0.7);
248        assert_eq!(meta.completeness, ResultCompleteness::Complete);
249    }
250
251    #[test]
252    fn test_scorer_registry() {
253        let registry = ScorerRegistry::new();
254        let result = json!({"files": ["a.txt", "b.txt"]});
255
256        let meta = registry.score("find", &result);
257        assert_eq!(meta.result_count, 2);
258    }
259
260    #[test]
261    fn test_command_session_aliases_share_shell_scorer() {
262        let registry = ScorerRegistry::new();
263        let result = json!({"exit_code": 0, "stdout": "line1\nline2"});
264
265        let unified_meta = registry.score(tools::UNIFIED_EXEC, &result);
266        let legacy_meta = registry.score(tools::RUN_PTY_CMD, &result);
267
268        assert_eq!(legacy_meta.result_count, unified_meta.result_count);
269        assert!((legacy_meta.confidence - unified_meta.confidence).abs() < f32::EPSILON);
270        assert!((legacy_meta.relevance - unified_meta.relevance).abs() < f32::EPSILON);
271    }
272}
vtcode_core/tools/result_metadata.rs

vtcode_core/tools/
result_metadata.rs
