1use hashbrown::HashMap;
10use serde::{Deserialize, Serialize};
11use std::fmt::Write;
12use std::time::Duration;
13use tracing::debug;
14
15#[cfg(test)]
16use crate::config::constants::tools;
17
18#[derive(Debug, Clone, Default, Serialize, Deserialize)]
20pub struct SkillStatistics {
21 pub creation_to_reuse_time: Option<Duration>,
23 pub avg_lifecycle: Option<Duration>,
25 pub reuse_ratio_by_tag: HashMap<String, f64>,
27 pub most_effective_skills: Vec<String>,
29 pub rarely_used_skills: Vec<String>,
31 pub total_skills: usize,
33 pub reused_skills: usize,
35}
36
37#[derive(Debug, Clone, Serialize, Deserialize)]
39pub struct ToolStatistics {
40 pub discovery_success_rate: f64,
42 pub usage_frequency: HashMap<String, u64>,
44 pub common_tool_chains: Vec<Vec<String>>,
46 pub typical_discovery_queries: Vec<String>,
48 pub total_discoveries: u64,
50 pub successful_discoveries: u64,
52}
53
54impl Default for ToolStatistics {
55 fn default() -> Self {
56 Self {
57 discovery_success_rate: 0.0,
58 usage_frequency: HashMap::new(),
59 common_tool_chains: vec![],
60 typical_discovery_queries: vec![],
61 total_discoveries: 0,
62 successful_discoveries: 0,
63 }
64 }
65}
66
67#[derive(Debug, Clone, Serialize, Deserialize)]
69pub struct CodePattern {
70 pub language: String,
72 pub pattern: String,
74 pub failure_rate: f64,
76 pub example_failures: Vec<String>,
78 pub occurrences: u64,
80}
81
82#[derive(Debug, Clone, Serialize, Deserialize)]
84pub struct RecoveryPattern {
85 pub error_type: String,
87 pub recovery_action: String,
89 pub success_rate: f64,
91 pub attempts: u64,
93}
94
95#[derive(Debug, Clone, Serialize, Deserialize)]
97pub struct AppliedRecovery {
98 pub error_type: String,
100 pub recovery_action: String,
102 pub success_rate: f64,
104 pub attempts: u64,
106}
107
108#[derive(Debug, Clone, Serialize, Deserialize, Default)]
110pub struct FailurePatterns {
111 pub high_failure_tools: Vec<(String, f64)>,
113 pub high_failure_patterns: Vec<CodePattern>,
115 pub common_errors: Vec<(String, u64)>,
117 pub recovery_patterns: Vec<RecoveryPattern>,
119}
120
121#[derive(Default)]
123pub struct AgentBehaviorAnalyzer {
124 skill_stats: SkillStatistics,
125 tool_stats: ToolStatistics,
126 failure_patterns: FailurePatterns,
127}
128
129impl AgentBehaviorAnalyzer {
130 pub fn new() -> Self {
132 Self::default()
133 }
134
135 pub fn skill_stats(&self) -> &SkillStatistics {
137 &self.skill_stats
138 }
139
140 pub fn tool_stats(&self) -> &ToolStatistics {
142 &self.tool_stats
143 }
144
145 pub fn failure_patterns(&self) -> &FailurePatterns {
147 &self.failure_patterns
148 }
149
150 pub fn recommend_tools(&self, query: &str, limit: usize) -> Vec<String> {
152 let mut recommendations = vec![];
153 let query_lower = query.to_lowercase();
154
155 for (tool, _count) in self.tool_stats.usage_frequency.iter().take(limit) {
157 if tool.to_lowercase().contains(&query_lower) {
158 recommendations.push(tool.clone());
159 }
160 }
161
162 if recommendations.is_empty() {
164 let mut by_usage: Vec<_> = self.tool_stats.usage_frequency.iter().collect();
165 by_usage.sort_by(|a, b| b.1.cmp(a.1));
166 recommendations = by_usage
167 .iter()
168 .take(limit)
169 .map(|pair| pair.0.clone())
170 .collect();
171 }
172
173 recommendations
174 }
175
176 pub fn recommend_skills(&self, limit: usize) -> Vec<String> {
178 self.skill_stats
179 .most_effective_skills
180 .iter()
181 .take(limit)
182 .cloned()
183 .collect()
184 }
185
186 pub fn identify_risky_tools(&self, failure_threshold: f64) -> Vec<(String, f64)> {
188 self.failure_patterns
189 .high_failure_tools
190 .iter()
191 .filter(|(_tool, rate)| *rate >= failure_threshold)
192 .cloned()
193 .collect()
194 }
195
196 pub fn get_recovery_strategy(&self, error_type: &str) -> Option<RecoveryPattern> {
198 self.failure_patterns
199 .recovery_patterns
200 .iter()
201 .find(|p| p.error_type == error_type)
202 .cloned()
203 }
204
205 pub fn record_tool_usage(&mut self, tool_name: &str) {
207 *self
208 .tool_stats
209 .usage_frequency
210 .entry(tool_name.into())
211 .or_insert(0) += 1;
212 }
213
214 pub fn record_skill_reuse(&mut self, skill_name: &str) {
216 if let Some(pos) = self
217 .skill_stats
218 .most_effective_skills
219 .iter()
220 .position(|s| s == skill_name)
221 {
222 let skill = self.skill_stats.most_effective_skills.remove(pos);
224 self.skill_stats.most_effective_skills.insert(0, skill);
225 } else {
226 self.skill_stats
227 .most_effective_skills
228 .insert(0, skill_name.into());
229 }
230 self.skill_stats.reused_skills += 1;
231 }
232
233 pub fn record_tool_failure(&mut self, tool_name: &str, error_msg: &str) {
235 if let Some(pos) = self
237 .failure_patterns
238 .common_errors
239 .iter()
240 .position(|(msg, _)| msg == error_msg)
241 {
242 self.failure_patterns.common_errors[pos].1 += 1;
243 } else {
244 self.failure_patterns
245 .common_errors
246 .push((error_msg.into(), 1));
247 }
248
249 let count = self
251 .failure_patterns
252 .common_errors
253 .iter()
254 .find(|(msg, _)| msg == error_msg)
255 .map(|(_, c)| *c)
256 .unwrap_or(1);
257 let failure_rate = count as f64 / (count + 1) as f64; if let Some(pos) = self
260 .failure_patterns
261 .high_failure_tools
262 .iter()
263 .position(|t| t.0 == tool_name)
264 {
265 self.failure_patterns.high_failure_tools[pos].1 = failure_rate;
266 } else {
267 self.failure_patterns
268 .high_failure_tools
269 .push((tool_name.into(), failure_rate));
270 }
271
272 debug!(
273 "Recorded failure for {}: {} (failure_rate: {})",
274 tool_name, error_msg, failure_rate
275 );
276 }
277
278 pub fn should_warn(&self, tool_name: &str) -> Option<String> {
280 for (tool, rate) in &self.failure_patterns.high_failure_tools {
281 if tool == tool_name && *rate >= 0.5 {
282 return Some(format!(
283 "Tool '{}' has a high failure rate ({:.1}%). Consider alternative approaches.",
284 tool_name,
285 rate * 100.0
286 ));
287 }
288 }
289 None
290 }
291
292 pub fn get_recovery_action(&self, error_type: &str) -> Option<String> {
294 self.failure_patterns
295 .recovery_patterns
296 .iter()
297 .find(|p| p.error_type == error_type)
298 .map(|p| {
299 format!(
300 "{} (success rate: {:.1}%)",
301 p.recovery_action,
302 p.success_rate * 100.0
303 )
304 })
305 }
306
307 pub fn export_metrics(&self) -> HashMap<String, serde_json::Value> {
309 let mut metrics = HashMap::new();
310
311 metrics.insert(
313 "total_skills".to_string(),
314 serde_json::json!(self.skill_stats.total_skills),
315 );
316 metrics.insert(
317 "reused_skills".to_string(),
318 serde_json::json!(self.skill_stats.reused_skills),
319 );
320
321 metrics.insert(
323 "discovery_success_rate".to_string(),
324 serde_json::json!(self.tool_stats.discovery_success_rate),
325 );
326 metrics.insert(
327 "total_tools_used".to_string(),
328 serde_json::json!(self.tool_stats.usage_frequency.len()),
329 );
330
331 metrics.insert(
333 "high_failure_tools_count".to_string(),
334 serde_json::json!(self.failure_patterns.high_failure_tools.len()),
335 );
336 metrics.insert(
337 "common_errors_count".to_string(),
338 serde_json::json!(self.failure_patterns.common_errors.len()),
339 );
340 metrics.insert(
341 "recovery_patterns_count".to_string(),
342 serde_json::json!(self.failure_patterns.recovery_patterns.len()),
343 );
344
345 let mut tool_usage: Vec<_> = self.tool_stats.usage_frequency.iter().collect();
347 tool_usage.sort_by(|a, b| b.1.cmp(a.1));
348 let top_tools: HashMap<String, u64> = tool_usage
349 .into_iter()
350 .take(10)
351 .map(|(k, v)| (k.clone(), *v))
352 .collect();
353 metrics.insert("top_tools".to_string(), serde_json::json!(top_tools));
354
355 metrics
356 }
357
358 pub fn tool_usage_count(&self, tool_name: &str) -> u64 {
360 *self.tool_stats.usage_frequency.get(tool_name).unwrap_or(&0)
361 }
362
363 pub fn tool_failure_rate(&self, tool_name: &str) -> f64 {
365 self.failure_patterns
366 .high_failure_tools
367 .iter()
368 .find(|(tool, _)| tool == tool_name)
369 .map(|(_, rate)| *rate)
370 .unwrap_or(0.0)
371 }
372
373 pub fn tool_success_rate(&self, tool_name: &str) -> f64 {
375 let usage = self.tool_usage_count(tool_name);
376 if usage == 0 {
377 return 1.0;
378 }
379
380 let failure_rate = self.tool_failure_rate(tool_name).clamp(0.0, 1.0);
381 (1.0 - failure_rate).max(0.0)
382 }
383
384 pub fn apply_recovery_pattern(&mut self, error_type: &str) -> Option<AppliedRecovery> {
387 let pattern = self
389 .failure_patterns
390 .recovery_patterns
391 .iter()
392 .find(|p| p.error_type == error_type)?;
393
394 let applied = AppliedRecovery {
396 error_type: error_type.to_owned(),
397 recovery_action: pattern.recovery_action.clone(),
398 success_rate: pattern.success_rate,
399 attempts: pattern.attempts,
400 };
401
402 debug!(
403 "Applying recovery pattern for '{}': {} (success rate: {:.1}%)",
404 error_type,
405 pattern.recovery_action,
406 pattern.success_rate * 100.0
407 );
408
409 Some(applied)
410 }
411
412 pub fn record_recovery_outcome(&mut self, error_type: &str, success: bool) {
414 if let Some(pattern) = self
415 .failure_patterns
416 .recovery_patterns
417 .iter_mut()
418 .find(|p| p.error_type == error_type)
419 {
420 pattern.attempts += 1;
421 if success {
422 let alpha = 0.3; pattern.success_rate = alpha + (1.0 - alpha) * pattern.success_rate;
425 } else {
426 let alpha = 0.3;
428 pattern.success_rate *= 1.0 - alpha;
429 }
430
431 debug!(
432 "Updated recovery pattern '{}': success_rate={:.1}%, attempts={}",
433 error_type,
434 pattern.success_rate * 100.0,
435 pattern.attempts
436 );
437 }
438 }
439
440 pub fn add_recovery_pattern(
442 &mut self,
443 error_type: String,
444 recovery_action: String,
445 initial_success_rate: f64,
446 ) {
447 if let Some(pattern) = self
449 .failure_patterns
450 .recovery_patterns
451 .iter_mut()
452 .find(|p| p.error_type == error_type)
453 {
454 pattern.recovery_action = recovery_action;
456 pattern.success_rate = initial_success_rate;
457 } else {
458 self.failure_patterns
460 .recovery_patterns
461 .push(RecoveryPattern {
462 error_type,
463 recovery_action,
464 success_rate: initial_success_rate,
465 attempts: 0,
466 });
467 }
468 }
469
470 pub fn summary(&self) -> String {
472 let mut output = String::new();
473 output.push_str("=== Agent Behavior Analysis ===\n\n");
474
475 output.push_str("## Skill Statistics\n");
476 let _ = writeln!(output, "Total skills: {}", self.skill_stats.total_skills);
477 let _ = writeln!(output, "Reused skills: {}", self.skill_stats.reused_skills);
478 if let Some(top_skill) = self.skill_stats.most_effective_skills.first() {
479 let _ = writeln!(output, "Top skill: {}", top_skill);
480 }
481
482 output.push_str("\n## Tool Statistics\n");
483 let _ = writeln!(
484 output,
485 "Tool discovery success rate: {:.1}%",
486 self.tool_stats.discovery_success_rate * 100.0
487 );
488 let _ = writeln!(
489 output,
490 "Total tools used: {}",
491 self.tool_stats.usage_frequency.len()
492 );
493
494 if !self.failure_patterns.high_failure_tools.is_empty() {
495 output.push_str("\n## High-Risk Tools\n");
496 for (tool, rate) in self.failure_patterns.high_failure_tools.iter().take(5) {
497 let _ = writeln!(output, "- {} (failure rate: {:.1}%)", tool, rate * 100.0);
498 }
499 }
500
501 output
502 }
503}
504
505#[cfg(test)]
506mod tests {
507 use super::*;
508
509 #[test]
510 fn test_analyzer_creation() {
511 let analyzer = AgentBehaviorAnalyzer::new();
512 assert_eq!(analyzer.skill_stats.total_skills, 0);
513 assert_eq!(analyzer.tool_stats.total_discoveries, 0);
514 }
515
516 #[test]
517 fn test_recommend_tools() {
518 let mut analyzer = AgentBehaviorAnalyzer::new();
519 analyzer.record_tool_usage("read_file");
520 analyzer.record_tool_usage("read_file");
521 analyzer.record_tool_usage("write_file");
522 analyzer.record_tool_usage(tools::LIST_FILES);
523
524 let recommendations = analyzer.recommend_tools("read", 1);
525 assert!(recommendations.contains(&"read_file".to_owned()));
526 }
527
528 #[test]
529 fn test_record_skill_reuse() {
530 let mut analyzer = AgentBehaviorAnalyzer::new();
531 analyzer.record_skill_reuse("filter_skill");
532 analyzer.record_skill_reuse("filter_skill");
533 analyzer.record_skill_reuse("transform_skill");
534
535 assert_eq!(analyzer.skill_stats.reused_skills, 3);
536 assert!(
537 analyzer
538 .skill_stats
539 .most_effective_skills
540 .contains(&"filter_skill".to_owned())
541 );
542 }
543
544 #[test]
545 fn test_tool_failure_tracking() {
546 let mut analyzer = AgentBehaviorAnalyzer::new();
547 analyzer.record_tool_failure(tools::GREP_FILE, "timeout");
548 analyzer.record_tool_failure(tools::GREP_FILE, "timeout");
549 analyzer.record_tool_failure(tools::GREP_FILE, "pattern_error");
550
551 assert!(!analyzer.failure_patterns.high_failure_tools.is_empty());
552 assert!(analyzer.failure_patterns.high_failure_tools[0].0 == tools::GREP_FILE);
553 }
554
555 #[test]
556 fn test_summary_generation() {
557 let mut analyzer = AgentBehaviorAnalyzer::new();
558 analyzer.skill_stats.total_skills = 5;
559 analyzer.record_skill_reuse("test_skill");
560
561 let summary = analyzer.summary();
562 assert!(summary.contains("Skill Statistics"));
563 assert!(summary.contains("Total skills: 5"));
564 assert!(summary.contains("Reused skills: 1"));
565 }
566
567 #[test]
568 fn test_identify_risky_tools() {
569 let mut analyzer = AgentBehaviorAnalyzer::new();
570 analyzer
571 .failure_patterns
572 .high_failure_tools
573 .push(("risky_tool".to_owned(), 0.8));
574 analyzer
575 .failure_patterns
576 .high_failure_tools
577 .push(("safe_tool".to_owned(), 0.1));
578
579 let risky = analyzer.identify_risky_tools(0.5);
580 assert_eq!(risky.len(), 1);
581 assert_eq!(risky[0].0, "risky_tool");
582 }
583
584 #[test]
585 fn test_recovery_pattern_lookup() {
586 let mut analyzer = AgentBehaviorAnalyzer::new();
587 analyzer
588 .failure_patterns
589 .recovery_patterns
590 .push(RecoveryPattern {
591 error_type: "timeout".to_owned(),
592 recovery_action: "retry with increased timeout".to_owned(),
593 success_rate: 0.85,
594 attempts: 20,
595 });
596
597 let recovery = analyzer.get_recovery_strategy("timeout");
598 assert!(recovery.is_some());
599 assert_eq!(recovery.unwrap().success_rate, 0.85);
600 }
601
602 #[test]
603 fn test_should_warn() {
604 let mut analyzer = AgentBehaviorAnalyzer::new();
605 analyzer
606 .failure_patterns
607 .high_failure_tools
608 .push(("risky_tool".to_owned(), 0.7));
609
610 let warning = analyzer.should_warn("risky_tool");
611 assert!(warning.is_some());
612 assert!(warning.unwrap().contains("high failure rate"));
613
614 let no_warning = analyzer.should_warn("safe_tool");
615 assert!(no_warning.is_none());
616 }
617
618 #[test]
619 fn test_get_recovery_action() {
620 let mut analyzer = AgentBehaviorAnalyzer::new();
621 analyzer
622 .failure_patterns
623 .recovery_patterns
624 .push(RecoveryPattern {
625 error_type: "network_error".to_owned(),
626 recovery_action: "retry with exponential backoff".to_owned(),
627 success_rate: 0.9,
628 attempts: 15,
629 });
630
631 let action = analyzer.get_recovery_action("network_error");
632 assert!(action.is_some());
633 let action_str = action.unwrap();
634 assert!(action_str.contains("retry with exponential backoff"));
635 assert!(action_str.contains("90.0%"));
636 }
637
638 #[test]
639 fn test_export_metrics() {
640 let mut analyzer = AgentBehaviorAnalyzer::new();
641 analyzer.skill_stats.total_skills = 10;
642 analyzer.skill_stats.reused_skills = 5;
643 analyzer.record_tool_usage("test_tool");
644
645 let metrics = analyzer.export_metrics();
646 assert_eq!(metrics.get("total_skills").unwrap(), &serde_json::json!(10));
647 assert_eq!(metrics.get("reused_skills").unwrap(), &serde_json::json!(5));
648 assert_eq!(
649 metrics.get("total_tools_used").unwrap(),
650 &serde_json::json!(1)
651 );
652 }
653
654 #[test]
655 fn test_apply_recovery_pattern() {
656 let mut analyzer = AgentBehaviorAnalyzer::new();
657 analyzer
658 .failure_patterns
659 .recovery_patterns
660 .push(RecoveryPattern {
661 error_type: "timeout".to_owned(),
662 recovery_action: "retry with increased timeout".to_owned(),
663 success_rate: 0.85,
664 attempts: 20,
665 });
666
667 let applied = analyzer.apply_recovery_pattern("timeout");
668 assert!(applied.is_some());
669 let applied = applied.unwrap();
670 assert_eq!(applied.error_type, "timeout");
671 assert_eq!(applied.recovery_action, "retry with increased timeout");
672 assert_eq!(applied.success_rate, 0.85);
673 assert_eq!(applied.attempts, 20);
674
675 let no_pattern = analyzer.apply_recovery_pattern("unknown_error");
677 assert!(no_pattern.is_none());
678 }
679
680 #[test]
681 fn test_record_recovery_outcome_success() {
682 let mut analyzer = AgentBehaviorAnalyzer::new();
683 analyzer
684 .failure_patterns
685 .recovery_patterns
686 .push(RecoveryPattern {
687 error_type: "network_error".to_owned(),
688 recovery_action: "retry".to_owned(),
689 success_rate: 0.5,
690 attempts: 10,
691 });
692
693 analyzer.record_recovery_outcome("network_error", true);
694
695 let pattern = &analyzer.failure_patterns.recovery_patterns[0];
696 assert_eq!(pattern.attempts, 11);
697 assert!(pattern.success_rate > 0.5);
699 }
700
701 #[test]
702 fn test_record_recovery_outcome_failure() {
703 let mut analyzer = AgentBehaviorAnalyzer::new();
704 analyzer
705 .failure_patterns
706 .recovery_patterns
707 .push(RecoveryPattern {
708 error_type: "parse_error".to_owned(),
709 recovery_action: "simplify input".to_owned(),
710 success_rate: 0.8,
711 attempts: 5,
712 });
713
714 analyzer.record_recovery_outcome("parse_error", false);
715
716 let pattern = &analyzer.failure_patterns.recovery_patterns[0];
717 assert_eq!(pattern.attempts, 6);
718 assert!(pattern.success_rate < 0.8);
720 }
721
722 #[test]
723 fn test_add_recovery_pattern_new() {
724 let mut analyzer = AgentBehaviorAnalyzer::new();
725
726 analyzer.add_recovery_pattern(
727 "new_error".to_owned(),
728 "new recovery action".to_owned(),
729 0.75,
730 );
731
732 assert_eq!(analyzer.failure_patterns.recovery_patterns.len(), 1);
733 let pattern = &analyzer.failure_patterns.recovery_patterns[0];
734 assert_eq!(pattern.error_type, "new_error");
735 assert_eq!(pattern.recovery_action, "new recovery action");
736 assert_eq!(pattern.success_rate, 0.75);
737 assert_eq!(pattern.attempts, 0);
738 }
739
740 #[test]
741 fn test_add_recovery_pattern_update_existing() {
742 let mut analyzer = AgentBehaviorAnalyzer::new();
743 analyzer
744 .failure_patterns
745 .recovery_patterns
746 .push(RecoveryPattern {
747 error_type: "existing_error".to_owned(),
748 recovery_action: "old action".to_owned(),
749 success_rate: 0.5,
750 attempts: 10,
751 });
752
753 analyzer.add_recovery_pattern(
754 "existing_error".to_owned(),
755 "updated action".to_owned(),
756 0.9,
757 );
758
759 assert_eq!(analyzer.failure_patterns.recovery_patterns.len(), 1);
760 let pattern = &analyzer.failure_patterns.recovery_patterns[0];
761 assert_eq!(pattern.error_type, "existing_error");
762 assert_eq!(pattern.recovery_action, "updated action");
763 assert_eq!(pattern.success_rate, 0.9);
764 assert_eq!(pattern.attempts, 10);
766 }
767
768 #[test]
769 fn test_export_metrics_with_recovery_patterns() {
770 let mut analyzer = AgentBehaviorAnalyzer::new();
771 analyzer.add_recovery_pattern("error1".to_owned(), "action1".to_owned(), 0.8);
772 analyzer.add_recovery_pattern("error2".to_owned(), "action2".to_owned(), 0.9);
773
774 let metrics = analyzer.export_metrics();
775 assert_eq!(
776 metrics.get("recovery_patterns_count").unwrap(),
777 &serde_json::json!(2)
778 );
779 }
780
781 #[test]
782 fn test_export_metrics_with_top_tools() {
783 let mut analyzer = AgentBehaviorAnalyzer::new();
784 analyzer.record_tool_usage("tool_a");
785 analyzer.record_tool_usage("tool_a");
786 analyzer.record_tool_usage("tool_a");
787 analyzer.record_tool_usage("tool_b");
788 analyzer.record_tool_usage("tool_b");
789 analyzer.record_tool_usage("tool_c");
790
791 let metrics = analyzer.export_metrics();
792 let top_tools = metrics.get("top_tools").unwrap();
793
794 assert!(top_tools.is_object());
796
797 let top_tools_map = top_tools.as_object().unwrap();
799 assert_eq!(top_tools_map.get("tool_a").unwrap(), &serde_json::json!(3));
800 assert_eq!(top_tools_map.get("tool_b").unwrap(), &serde_json::json!(2));
801 assert_eq!(top_tools_map.get("tool_c").unwrap(), &serde_json::json!(1));
802 }
803}