1use std::{
2 collections::{HashMap, HashSet},
3 path::Path,
4};
5
6use anyhow::Result;
7
8use crate::{
9 cache, embeddings, extract,
10 models::{
11 Candidate, EvalMatrixReport, EvalReport, FunctionRecord, FunctionSummary, ModelEvalResult,
12 ModelEvalStatus, OutputFormat, Report, ReportConfig, StrategyEval,
13 },
14 score,
15};
16
17pub fn run_report(path: &Path, config: &ReportConfig) -> Result<Report> {
18 let (project_root, functions) = extract::extract_functions(path, config)?;
19 build_report(project_root, functions, config, config.top_k)
20}
21
22pub fn run_eval(path: &Path, config: &ReportConfig) -> Result<EvalReport> {
23 let display_top_k = config.top_k;
24 let (project_root, functions) = extract::extract_functions(path, config)?;
25 let known_pairs = known_pair_count(&functions);
26 let function_summaries = function_summaries(&functions);
27 let (report, all_candidates) =
28 build_eval_report(project_root, functions, config, display_top_k)?;
29
30 Ok(EvalReport {
31 functions: function_summaries,
32 clone: strategy_eval(&all_candidates, known_pairs, |candidate| {
33 candidate.scores.clone_flag
34 }),
35 semantic: strategy_eval(&all_candidates, known_pairs, |candidate| {
36 candidate.scores.semantic_flag
37 }),
38 hybrid: strategy_eval(&all_candidates, known_pairs, |candidate| {
39 candidate.scores.hybrid_flag
40 }),
41 report,
42 })
43}
44
45pub fn run_eval_matrix(
46 path: &Path,
47 config: &ReportConfig,
48 models: &[String],
49) -> Result<EvalMatrixReport> {
50 let display_top_k = config.top_k;
51 let (project_root, functions) = extract::extract_functions(path, config)?;
52 let known_pairs = known_pair_count(&functions);
53 let function_summaries = function_summaries(&functions);
54 let mut results = Vec::with_capacity(models.len());
55
56 for model in models {
57 let mut model_config = config.clone();
58 model_config.model = Some(model.clone());
59 match build_eval_report(
60 project_root.clone(),
61 functions.clone(),
62 &model_config,
63 display_top_k,
64 ) {
65 Ok((report, all_candidates)) => {
66 results.push(ModelEvalResult {
67 model: model.clone(),
68 status: ModelEvalStatus::Success,
69 error_kind: None,
70 error: None,
71 clone: Some(strategy_eval(&all_candidates, known_pairs, |candidate| {
72 candidate.scores.clone_flag
73 })),
74 semantic: Some(strategy_eval(&all_candidates, known_pairs, |candidate| {
75 candidate.scores.semantic_flag
76 })),
77 hybrid: Some(strategy_eval(&all_candidates, known_pairs, |candidate| {
78 candidate.scores.hybrid_flag
79 })),
80 report: Some(report),
81 });
82 }
83 Err(error) => {
84 results.push(ModelEvalResult {
85 model: model.clone(),
86 status: ModelEvalStatus::Failure,
87 error_kind: Some(classify_eval_error(&error).to_owned()),
88 error: Some(error.to_string()),
89 report: None,
90 clone: None,
91 semantic: None,
92 hybrid: None,
93 });
94 }
95 }
96 }
97
98 Ok(EvalMatrixReport {
99 project_root,
100 provider: config.provider.as_str().to_owned(),
101 functions_count: functions.len(),
102 known_pairs,
103 threshold: config.threshold,
104 top_k: display_top_k,
105 functions: function_summaries,
106 models: results,
107 })
108}
109
110fn build_report(
111 project_root: std::path::PathBuf,
112 functions: Vec<crate::models::FunctionRecord>,
113 config: &ReportConfig,
114 top_k: usize,
115) -> Result<Report> {
116 let cache_root = cache::cache_root(&project_root, config.cache_dir.as_deref());
117 let (embeddings, embedding_stats) =
118 embeddings::embeddings_for(&functions, config, &cache_root)?;
119 let candidates = score::score_candidates(&functions, &embeddings, config.threshold, top_k);
120
121 Ok(Report {
122 project_root,
123 provider: config.provider.as_str().to_owned(),
124 model: config.model.clone(),
125 functions_count: functions.len(),
126 embedding_stats,
127 candidates,
128 })
129}
130
131fn build_eval_report(
132 project_root: std::path::PathBuf,
133 functions: Vec<crate::models::FunctionRecord>,
134 config: &ReportConfig,
135 display_top_k: usize,
136) -> Result<(Report, Vec<Candidate>)> {
137 let cache_root = cache::cache_root(&project_root, config.cache_dir.as_deref());
138 let (embeddings, embedding_stats) =
139 embeddings::embeddings_for(&functions, config, &cache_root)?;
140 let all_candidates =
141 score::score_candidates(&functions, &embeddings, config.threshold, usize::MAX);
142 let mut display_candidates = all_candidates.clone();
143 display_candidates.truncate(display_top_k);
144
145 let report = Report {
146 project_root,
147 provider: config.provider.as_str().to_owned(),
148 model: config.model.clone(),
149 functions_count: functions.len(),
150 embedding_stats,
151 candidates: display_candidates,
152 };
153
154 Ok((report, all_candidates))
155}
156
157pub fn format_report(report: &Report, format: OutputFormat) -> Result<String> {
158 match format {
159 OutputFormat::Json => Ok(serde_json::to_string_pretty(report)?),
160 OutputFormat::Markdown => Ok(format_markdown(report)),
161 OutputFormat::Table => Ok(format_table(report)),
162 }
163}
164
165pub fn format_eval(eval: &EvalReport, format: OutputFormat) -> Result<String> {
166 match format {
167 OutputFormat::Json => Ok(serde_json::to_string_pretty(eval)?),
168 OutputFormat::Markdown => {
169 let mut out = String::new();
170 out.push_str("# Funcvec Eval\n\n");
171 out.push_str(&format!(
172 "- functions: {}\n- candidates: {}\n\n",
173 eval.report.functions_count,
174 eval.report.candidates.len()
175 ));
176 out.push_str(
177 "| strategy | flagged | true positives | false positives | known pairs | precision | recall | f1 |\n",
178 );
179 out.push_str("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n");
180 write_eval_row(&mut out, "clone", &eval.clone);
181 write_eval_row(&mut out, "semantic", &eval.semantic);
182 write_eval_row(&mut out, "hybrid", &eval.hybrid);
183 out.push_str("\n## Functions\n\n");
184 append_functions_markdown(&mut out, &eval.functions);
185 out.push('\n');
186 out.push_str(&format_markdown(&eval.report));
187 Ok(out)
188 }
189 OutputFormat::Table => {
190 let mut out = String::new();
191 out.push_str("strategy flagged true+ false+ known prec recall f1\n");
192 out.push_str("-------------------------------------------------------------\n");
193 table_eval_row(&mut out, "clone", &eval.clone);
194 table_eval_row(&mut out, "semantic", &eval.semantic);
195 table_eval_row(&mut out, "hybrid", &eval.hybrid);
196 out.push('\n');
197 append_functions_table(&mut out, &eval.functions);
198 out.push('\n');
199 out.push_str(&format_table(&eval.report));
200 Ok(out)
201 }
202 }
203}
204
205pub fn format_eval_matrix(eval: &EvalMatrixReport, format: OutputFormat) -> Result<String> {
206 match format {
207 OutputFormat::Json => Ok(serde_json::to_string_pretty(eval)?),
208 OutputFormat::Markdown => Ok(format_eval_matrix_markdown(eval)),
209 OutputFormat::Table => Ok(format_eval_matrix_table(eval)),
210 }
211}
212
213fn classify_eval_error(error: &anyhow::Error) -> &'static str {
214 let text = error.to_string().to_ascii_lowercase();
215 if text.contains("could not connect") || text.contains("timed out") {
216 "daemon_unavailable"
217 } else if text.contains("ollama model") && text.contains("not available") {
218 "model_missing"
219 } else if text.contains("non-loopback") {
220 "nonlocal_host_rejected"
221 } else if text.contains("embedding") {
222 "provider_response"
223 } else {
224 "provider_error"
225 }
226}
227
228fn known_pair_count(functions: &[crate::models::FunctionRecord]) -> usize {
229 let mut groups: HashMap<&str, HashSet<&str>> = HashMap::new();
230 for function in functions {
231 if let Some(group) = function.expected_group.as_deref() {
232 groups.entry(group).or_default().insert(&function.id);
233 }
234 }
235 groups
236 .values()
237 .map(|ids| {
238 let count = ids.len();
239 count.saturating_sub(1) * count / 2
240 })
241 .sum()
242}
243
244fn function_summaries(functions: &[FunctionRecord]) -> Vec<FunctionSummary> {
245 functions.iter().map(FunctionSummary::from).collect()
246}
247
248fn strategy_eval(
249 candidates: &[Candidate],
250 known_pairs: usize,
251 flagged: impl Fn(&Candidate) -> bool,
252) -> StrategyEval {
253 let mut result = StrategyEval {
254 flagged: 0,
255 true_positives: 0,
256 false_positives: 0,
257 known_pairs,
258 precision: 0.0,
259 recall: 0.0,
260 f1: 0.0,
261 };
262 for candidate in candidates {
263 if !flagged(candidate) {
264 continue;
265 }
266 result.flagged += 1;
267 if candidate.expected_match {
268 result.true_positives += 1;
269 } else {
270 result.false_positives += 1;
271 }
272 }
273 result.precision = if result.flagged == 0 {
274 0.0
275 } else {
276 result.true_positives as f32 / result.flagged as f32
277 };
278 result.recall = if result.known_pairs == 0 {
279 0.0
280 } else {
281 result.true_positives as f32 / result.known_pairs as f32
282 };
283 result.f1 = if result.precision + result.recall == 0.0 {
284 0.0
285 } else {
286 2.0 * result.precision * result.recall / (result.precision + result.recall)
287 };
288 result
289}
290
291fn format_table(report: &Report) -> String {
292 let mut out = String::new();
293 out.push_str(&format!(
294 "project: {}\nprovider: {}\nfunctions: {}\ncandidates: {}\ncache: {} hit / {} miss\nembedding dims: {}\n\n",
295 report.project_root.display(),
296 report.provider,
297 report.functions_count,
298 report.candidates.len(),
299 report.embedding_stats.cache_hits,
300 report.embedding_stats.cache_misses,
301 report
302 .embedding_stats
303 .dimensions
304 .map(|value| value.to_string())
305 .unwrap_or_else(|| "--".to_owned())
306 ));
307 out.push_str("hybrid clone sem left -> right\n");
308 out.push_str("------------------------------------\n");
309 for candidate in &report.candidates {
310 let sem = candidate
311 .scores
312 .semantic
313 .map(|score| format!("{score:.2}"))
314 .unwrap_or_else(|| "--".to_owned());
315 out.push_str(&format!(
316 "{:.2} {:.2} {:<5} {}:{} {} -> {}:{} {}\n",
317 candidate.scores.hybrid,
318 candidate.scores.clone,
319 sem,
320 candidate.left.file.display(),
321 candidate.left.start_line,
322 candidate.left.name,
323 candidate.right.file.display(),
324 candidate.right.start_line,
325 candidate.right.name
326 ));
327 if !candidate.reasons.is_empty() {
328 out.push_str(&format!(
329 " reasons: {}\n",
330 candidate.reasons.join(", ")
331 ));
332 }
333 }
334 out
335}
336
337fn format_markdown(report: &Report) -> String {
338 let mut out = String::new();
339 out.push_str("# Funcvec Report\n\n");
340 out.push_str(&format!(
341 "- project: `{}`\n- provider: `{}`\n- functions: `{}`\n- candidates: `{}`\n- cache: `{}` hit / `{}` miss\n- embedding dims: `{}`\n\n",
342 report.project_root.display(),
343 report.provider,
344 report.functions_count,
345 report.candidates.len(),
346 report.embedding_stats.cache_hits,
347 report.embedding_stats.cache_misses,
348 report
349 .embedding_stats
350 .dimensions
351 .map(|value| value.to_string())
352 .unwrap_or_else(|| "--".to_owned())
353 ));
354 out.push_str("| hybrid | clone | semantic | left | right | reasons |\n");
355 out.push_str("| ---: | ---: | ---: | --- | --- | --- |\n");
356 for candidate in &report.candidates {
357 let sem = candidate
358 .scores
359 .semantic
360 .map(|score| format!("{score:.2}"))
361 .unwrap_or_else(|| "--".to_owned());
362 out.push_str(&format!(
363 "| {:.2} | {:.2} | {} | `{}`:{} `{}` | `{}`:{} `{}` | {} |\n",
364 candidate.scores.hybrid,
365 candidate.scores.clone,
366 sem,
367 candidate.left.file.display(),
368 candidate.left.start_line,
369 candidate.left.name,
370 candidate.right.file.display(),
371 candidate.right.start_line,
372 candidate.right.name,
373 candidate.reasons.join(", ")
374 ));
375 }
376 out
377}
378
379fn write_eval_row(out: &mut String, name: &str, eval: &StrategyEval) {
380 out.push_str(&format!(
381 "| {name} | {} | {} | {} | {} | {:.2} | {:.2} | {:.2} |\n",
382 eval.flagged,
383 eval.true_positives,
384 eval.false_positives,
385 eval.known_pairs,
386 eval.precision,
387 eval.recall,
388 eval.f1
389 ));
390}
391
392fn table_eval_row(out: &mut String, name: &str, eval: &StrategyEval) {
393 out.push_str(&format!(
394 "{name:<10} {:>7} {:>6} {:>7} {:>6} {:>5.2} {:>6.2} {:>4.2}\n",
395 eval.flagged,
396 eval.true_positives,
397 eval.false_positives,
398 eval.known_pairs,
399 eval.precision,
400 eval.recall,
401 eval.f1
402 ));
403}
404
405fn format_eval_matrix_table(eval: &EvalMatrixReport) -> String {
406 let mut out = String::new();
407 out.push_str(&format!(
408 "project: {}\nprovider: {}\nfunctions: {}\nknown pairs: {}\nthreshold: {:.2}\n\n",
409 eval.project_root.display(),
410 eval.provider,
411 eval.functions_count,
412 eval.known_pairs,
413 eval.threshold
414 ));
415 append_functions_table(&mut out, &eval.functions);
416 out.push('\n');
417 out.push_str(
418 "model strategy status dims hit miss flagged true+ false+ prec recall f1 error\n",
419 );
420 out.push_str(
421 "--------------------------------------------------------------------------------------------------------\n",
422 );
423 for result in &eval.models {
424 match (
425 &result.status,
426 &result.report,
427 &result.clone,
428 &result.semantic,
429 &result.hybrid,
430 ) {
431 (ModelEvalStatus::Success, Some(report), Some(clone), Some(semantic), Some(hybrid)) => {
432 table_eval_matrix_row(&mut out, &result.model, "clone", report, clone);
433 table_eval_matrix_row(&mut out, &result.model, "semantic", report, semantic);
434 table_eval_matrix_row(&mut out, &result.model, "hybrid", report, hybrid);
435 }
436 _ => {
437 out.push_str(&format!(
438 "{:<29} {:<10} failure {:>4} {:>5} {:>5} {:>7} {:>6} {:>6} {:>4} {:>6} {:>4} {}\n",
439 result.model,
440 "-",
441 "--",
442 0,
443 0,
444 0,
445 0,
446 0,
447 "--",
448 "--",
449 "--",
450 result.error.as_deref().unwrap_or("unknown provider error")
451 ));
452 }
453 }
454 }
455 out
456}
457
458fn table_eval_matrix_row(
459 out: &mut String,
460 model: &str,
461 strategy: &str,
462 report: &Report,
463 eval: &StrategyEval,
464) {
465 out.push_str(&format!(
466 "{model:<29} {strategy:<10} success {dims:>4} {hit:>5} {miss:>5} {flagged:>7} {true_pos:>6} {false_pos:>6} {precision:>4.2} {recall:>6.2} {f1:>4.2} \n",
467 dims = report
468 .embedding_stats
469 .dimensions
470 .map(|value| value.to_string())
471 .unwrap_or_else(|| "--".to_owned()),
472 hit = report.embedding_stats.cache_hits,
473 miss = report.embedding_stats.cache_misses,
474 flagged = eval.flagged,
475 true_pos = eval.true_positives,
476 false_pos = eval.false_positives,
477 precision = eval.precision,
478 recall = eval.recall,
479 f1 = eval.f1
480 ));
481}
482
483fn format_eval_matrix_markdown(eval: &EvalMatrixReport) -> String {
484 let mut out = String::new();
485 out.push_str("# Funcvec Eval Matrix\n\n");
486 out.push_str(&format!(
487 "- project: `{}`\n- provider: `{}`\n- functions: `{}`\n- known pairs: `{}`\n- threshold: `{:.2}`\n\n",
488 eval.project_root.display(),
489 eval.provider,
490 eval.functions_count,
491 eval.known_pairs,
492 eval.threshold
493 ));
494 out.push_str("## Functions\n\n");
495 append_functions_markdown(&mut out, &eval.functions);
496 out.push('\n');
497 out.push_str("| model | strategy | status | dims | cache hit | cache miss | flagged | true+ | false+ | precision | recall | f1 | error |\n");
498 out.push_str("| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |\n");
499 for result in &eval.models {
500 match (
501 &result.status,
502 &result.report,
503 &result.clone,
504 &result.semantic,
505 &result.hybrid,
506 ) {
507 (ModelEvalStatus::Success, Some(report), Some(clone), Some(semantic), Some(hybrid)) => {
508 markdown_eval_matrix_row(&mut out, &result.model, "clone", report, clone);
509 markdown_eval_matrix_row(&mut out, &result.model, "semantic", report, semantic);
510 markdown_eval_matrix_row(&mut out, &result.model, "hybrid", report, hybrid);
511 }
512 _ => {
513 out.push_str(&format!(
514 "| `{}` | - | failure | -- | 0 | 0 | 0 | 0 | 0 | -- | -- | -- | {} |\n",
515 result.model,
516 result.error.as_deref().unwrap_or("unknown provider error")
517 ));
518 }
519 }
520 }
521 out
522}
523
524fn markdown_eval_matrix_row(
525 out: &mut String,
526 model: &str,
527 strategy: &str,
528 report: &Report,
529 eval: &StrategyEval,
530) {
531 out.push_str(&format!(
532 "| `{model}` | {strategy} | success | {dims} | {hit} | {miss} | {flagged} | {true_pos} | {false_pos} | {precision:.2} | {recall:.2} | {f1:.2} | |\n",
533 dims = report
534 .embedding_stats
535 .dimensions
536 .map(|value| value.to_string())
537 .unwrap_or_else(|| "--".to_owned()),
538 hit = report.embedding_stats.cache_hits,
539 miss = report.embedding_stats.cache_misses,
540 flagged = eval.flagged,
541 true_pos = eval.true_positives,
542 false_pos = eval.false_positives,
543 precision = eval.precision,
544 recall = eval.recall,
545 f1 = eval.f1
546 ));
547}
548
549fn append_functions_table(out: &mut String, functions: &[FunctionSummary]) {
550 out.push_str("functions\n");
551 out.push_str("---------\n");
552 for function in functions {
553 out.push_str(&format!(
554 "{}:{}-{} {:<34} lines={:<3} tokens={:<3} group={}\n",
555 function.file.display(),
556 function.start_line,
557 function.end_line,
558 function.name,
559 function.line_count,
560 function.token_count,
561 function.expected_group.as_deref().unwrap_or("-")
562 ));
563 }
564}
565
566fn append_functions_markdown(out: &mut String, functions: &[FunctionSummary]) {
567 out.push_str("| function | location | lines | tokens | expected group |\n");
568 out.push_str("| --- | --- | ---: | ---: | --- |\n");
569 for function in functions {
570 out.push_str(&format!(
571 "| `{}` | `{}`:{}-{} | {} | {} | {} |\n",
572 function.name,
573 function.file.display(),
574 function.start_line,
575 function.end_line,
576 function.line_count,
577 function.token_count,
578 function.expected_group.as_deref().unwrap_or("-")
579 ));
580 }
581}