llm_git/testing/
compare.rs

1//! Comparison logic for fixture testing
2
3use crate::types::ConventionalAnalysis;
4
5/// Result of comparing actual output to golden
6#[derive(Debug, Clone)]
7pub struct CompareResult {
8   /// Whether the type matches
9   pub type_match:          bool,
10   /// Whether the scope matches (or both are None)
11   pub scope_match:         bool,
12   /// Scope difference description if any
13   pub scope_diff:          Option<String>,
14   /// Number of details in golden
15   pub golden_detail_count: usize,
16   /// Number of details in actual
17   pub actual_detail_count: usize,
18   /// Overall pass/fail
19   pub passed:              bool,
20   /// Human-readable summary
21   pub summary:             String,
22}
23
24/// Compare actual analysis to golden
25pub fn compare_analysis(
26   golden: &ConventionalAnalysis,
27   actual: &ConventionalAnalysis,
28) -> CompareResult {
29   let type_match = golden.commit_type == actual.commit_type;
30
31   let scope_match = golden.scope == actual.scope;
32   let scope_diff = if scope_match {
33      None
34   } else {
35      Some(format!(
36         "{} → {}",
37         golden.scope.as_ref().map_or("null", |s| s.as_str()),
38         actual.scope.as_ref().map_or("null", |s| s.as_str())
39      ))
40   };
41
42   let golden_detail_count = golden.details.len();
43   let actual_detail_count = actual.details.len();
44
45   // Type mismatch is a hard failure
46   // Scope mismatch is a warning (might be an improvement)
47   let passed = type_match;
48
49   let summary = if passed && scope_match {
50      format!(
51         "✓ {} | {} | {} details",
52         actual.commit_type.as_str(),
53         actual.scope.as_ref().map_or("(no scope)", |s| s.as_str()),
54         actual_detail_count
55      )
56   } else if passed {
57      format!(
58         "≈ {} | scope: {} | {} details",
59         actual.commit_type.as_str(),
60         scope_diff.as_ref().unwrap(),
61         actual_detail_count
62      )
63   } else {
64      format!(
65         "✗ type: {} → {} | {} details",
66         golden.commit_type.as_str(),
67         actual.commit_type.as_str(),
68         actual_detail_count
69      )
70   };
71
72   CompareResult {
73      type_match,
74      scope_match,
75      scope_diff,
76      golden_detail_count,
77      actual_detail_count,
78      passed,
79      summary,
80   }
81}
82
83#[cfg(test)]
84mod tests {
85   use std::collections::HashSet;
86
87   use super::*;
88   use crate::types::{CommitType, Scope};
89
90   /// Compute Jaccard similarity between two strings (word-level)
91   fn jaccard_similarity(a: &str, b: &str) -> f64 {
92      let words_a: HashSet<&str> = a.split_whitespace().collect();
93      let words_b: HashSet<&str> = b.split_whitespace().collect();
94
95      if words_a.is_empty() && words_b.is_empty() {
96         return 1.0;
97      }
98
99      let intersection = words_a.intersection(&words_b).count();
100      let union = words_a.union(&words_b).count();
101
102      if union == 0 {
103         return 0.0;
104      }
105
106      intersection as f64 / union as f64
107   }
108
109   #[test]
110   fn test_compare_exact_match() {
111      let golden = ConventionalAnalysis {
112         commit_type: CommitType::new("feat").unwrap(),
113         scope:       Some(Scope::new("api").unwrap()),
114         summary:     None,
115         details:     vec![],
116         issue_refs:  vec![],
117      };
118      let actual = golden.clone();
119
120      let result = compare_analysis(&golden, &actual);
121      assert!(result.passed);
122      assert!(result.type_match);
123      assert!(result.scope_match);
124   }
125
126   #[test]
127   fn test_compare_type_mismatch() {
128      let golden = ConventionalAnalysis {
129         commit_type: CommitType::new("feat").unwrap(),
130         scope:       None,
131         summary:     None,
132         details:     vec![],
133         issue_refs:  vec![],
134      };
135      let actual = ConventionalAnalysis {
136         commit_type: CommitType::new("fix").unwrap(),
137         scope:       None,
138         summary:     None,
139         details:     vec![],
140         issue_refs:  vec![],
141      };
142
143      let result = compare_analysis(&golden, &actual);
144      assert!(!result.passed);
145      assert!(!result.type_match);
146   }
147
148   #[test]
149   fn test_compare_scope_mismatch() {
150      let golden = ConventionalAnalysis {
151         commit_type: CommitType::new("feat").unwrap(),
152         scope:       Some(Scope::new("api").unwrap()),
153         summary:     None,
154         details:     vec![],
155         issue_refs:  vec![],
156      };
157      let actual = ConventionalAnalysis {
158         commit_type: CommitType::new("feat").unwrap(),
159         scope:       Some(Scope::new("api/client").unwrap()),
160         summary:     None,
161         details:     vec![],
162         issue_refs:  vec![],
163      };
164
165      let result = compare_analysis(&golden, &actual);
166      assert!(result.passed); // Scope mismatch is warning, not failure
167      assert!(!result.scope_match);
168      assert!(result.scope_diff.is_some());
169   }
170
171   #[test]
172   fn test_jaccard_similarity() {
173      assert!((jaccard_similarity("hello world", "hello world") - 1.0).abs() < 0.001);
174      assert!((jaccard_similarity("hello world", "hello there") - 0.333).abs() < 0.1);
175      assert!((jaccard_similarity("", "") - 1.0).abs() < 0.001);
176   }
177}
llm_git/testing/compare.rs

llm_git/testing/
compare.rs
