1use crate::{
2 config,
3 data::{Commit, MeasurementData},
4 measurement_retrieval::{self, summarize_measurements},
5 stats::{self, DispersionMethod, ReductionFunc, StatsWithUnit, VecAggregation},
6};
7use anyhow::{anyhow, bail, Result};
8use itertools::Itertools;
9use log::error;
10use sparklines::spark;
11use std::cmp::Ordering;
12use std::collections::HashSet;
13use std::iter;
14
15fn format_z_score_display(z_score: f64) -> String {
19 if z_score.is_finite() {
20 format!(" {:.2}", z_score)
21 } else {
22 String::new()
23 }
24}
25
26fn get_direction_arrow(head_mean: f64, tail_mean: f64) -> &'static str {
30 match head_mean.partial_cmp(&tail_mean) {
31 Some(Ordering::Greater) => "↑",
32 Some(Ordering::Less) => "↓",
33 Some(Ordering::Equal) | None => "→",
34 }
35}
36
37#[derive(Debug, PartialEq)]
38struct AuditResult {
39 message: String,
40 passed: bool,
41}
42
43#[derive(Debug, PartialEq)]
45pub(crate) struct ResolvedAuditParams {
46 pub min_count: u16,
47 pub summarize_by: ReductionFunc,
48 pub sigma: f64,
49 pub dispersion_method: DispersionMethod,
50}
51
52pub(crate) fn resolve_audit_params(
58 measurement: &str,
59 cli_min_count: Option<u16>,
60 cli_summarize_by: Option<ReductionFunc>,
61 cli_sigma: Option<f64>,
62 cli_dispersion_method: Option<DispersionMethod>,
63) -> ResolvedAuditParams {
64 let min_count = cli_min_count
65 .or_else(|| config::audit_min_measurements(measurement))
66 .unwrap_or(2);
67
68 let summarize_by = cli_summarize_by
69 .or_else(|| config::audit_aggregate_by(measurement).map(ReductionFunc::from))
70 .unwrap_or(ReductionFunc::Min);
71
72 let sigma = cli_sigma
73 .or_else(|| config::audit_sigma(measurement))
74 .unwrap_or(4.0);
75
76 let dispersion_method = cli_dispersion_method
77 .or_else(|| {
78 Some(DispersionMethod::from(config::audit_dispersion_method(
79 measurement,
80 )))
81 })
82 .unwrap_or(DispersionMethod::StandardDeviation);
83
84 ResolvedAuditParams {
85 min_count,
86 summarize_by,
87 sigma,
88 dispersion_method,
89 }
90}
91
92fn discover_matching_measurements(
95 commits: &[Result<Commit>],
96 filters: &[regex::Regex],
97 selectors: &[(String, String)],
98) -> Vec<String> {
99 let mut unique_measurements = HashSet::new();
100
101 for commit in commits.iter().flatten() {
102 for measurement in &commit.measurements {
103 if !crate::filter::matches_any_filter(&measurement.name, filters) {
105 continue;
106 }
107
108 if !measurement.key_values_is_superset_of(selectors) {
110 continue;
111 }
112
113 unique_measurements.insert(measurement.name.clone());
115 }
116 }
117
118 let mut result: Vec<String> = unique_measurements.into_iter().collect();
120 result.sort();
121 result
122}
123
124pub fn audit_multiple(
125 max_count: usize,
126 min_count: Option<u16>,
127 selectors: &[(String, String)],
128 summarize_by: Option<ReductionFunc>,
129 sigma: Option<f64>,
130 dispersion_method: Option<DispersionMethod>,
131 combined_patterns: &[String],
132) -> Result<()> {
133 if combined_patterns.is_empty() {
135 return Ok(());
136 }
137
138 let filters = crate::filter::compile_filters(combined_patterns)?;
141
142 let all_commits: Vec<Result<Commit>> =
145 measurement_retrieval::walk_commits(max_count)?.collect();
146
147 let measurements_to_audit = discover_matching_measurements(&all_commits, &filters, selectors);
150
151 if measurements_to_audit.is_empty() {
153 if all_commits.is_empty() {
155 bail!("No commit at HEAD");
156 }
157 let has_any_measurements = all_commits.iter().any(|commit_result| {
159 if let Ok(commit) = commit_result {
160 !commit.measurements.is_empty()
161 } else {
162 false
163 }
164 });
165
166 if !has_any_measurements {
167 bail!("No measurement for HEAD");
169 }
170 bail!("No measurements found matching the provided patterns");
172 }
173
174 let mut failed = false;
175
176 for measurement in measurements_to_audit {
178 let params = resolve_audit_params(
179 &measurement,
180 min_count,
181 summarize_by,
182 sigma,
183 dispersion_method,
184 );
185
186 if (max_count as u16) < params.min_count {
188 eprintln!(
189 "⚠️ Warning: --max_count ({}) is less than min_measurements ({}) for measurement '{}'.",
190 max_count, params.min_count, measurement
191 );
192 eprintln!(
193 " This limits available historical data and may prevent achieving statistical significance."
194 );
195 }
196
197 let result = audit_with_commits(
198 &measurement,
199 &all_commits,
200 params.min_count,
201 selectors,
202 params.summarize_by,
203 params.sigma,
204 params.dispersion_method,
205 )?;
206
207 println!("{}", result.message);
208
209 if !result.passed {
210 failed = true;
211 }
212 }
213
214 if failed {
215 bail!("One or more measurements failed audit.");
216 }
217
218 Ok(())
219}
220
221fn audit_with_commits(
225 measurement: &str,
226 commits: &[Result<Commit>],
227 min_count: u16,
228 selectors: &[(String, String)],
229 summarize_by: ReductionFunc,
230 sigma: f64,
231 dispersion_method: DispersionMethod,
232) -> Result<AuditResult> {
233 let commits_iter = commits.iter().map(|r| match r {
236 Ok(commit) => Ok(Commit {
237 commit: commit.commit.clone(),
238 measurements: commit.measurements.clone(),
239 }),
240 Err(e) => Err(anyhow::anyhow!("{}", e)),
241 });
242
243 let filter_by =
245 |m: &MeasurementData| m.name == measurement && m.key_values_is_superset_of(selectors);
246
247 let mut aggregates = measurement_retrieval::take_while_same_epoch(summarize_measurements(
248 commits_iter,
249 &summarize_by,
250 &filter_by,
251 ));
252
253 let head = aggregates
254 .next()
255 .ok_or(anyhow!("No commit at HEAD"))
256 .and_then(|s| {
257 s.and_then(|cs| {
258 cs.measurement
259 .map(|m| m.val)
260 .ok_or(anyhow!("No measurement for HEAD."))
261 })
262 })?;
263
264 let tail: Vec<_> = aggregates
265 .filter_map_ok(|cs| cs.measurement.map(|m| m.val))
266 .try_collect()?;
267
268 audit_with_data(measurement, head, tail, min_count, sigma, dispersion_method)
269}
270
271fn audit_with_data(
274 measurement: &str,
275 head: f64,
276 tail: Vec<f64>,
277 min_count: u16,
278 sigma: f64,
279 dispersion_method: DispersionMethod,
280) -> Result<AuditResult> {
281 let unit = config::measurement_unit(measurement);
283 let unit_str = unit.as_deref();
284
285 let head_summary = stats::aggregate_measurements(iter::once(&head));
286 let tail_summary = stats::aggregate_measurements(tail.iter());
287
288 let all_measurements = tail.into_iter().chain(iter::once(head)).collect::<Vec<_>>();
290
291 let mut tail_measurements = all_measurements.clone();
292 tail_measurements.pop(); let tail_median = tail_measurements.median().unwrap_or(0.0);
294
295 let min_val = all_measurements
297 .iter()
298 .min_by(|a, b| a.partial_cmp(b).unwrap())
299 .unwrap();
300 let max_val = all_measurements
301 .iter()
302 .max_by(|a, b| a.partial_cmp(b).unwrap())
303 .unwrap();
304
305 let tail_median_is_zero = tail_median.abs() < f64::EPSILON;
309
310 let sparkline = if tail_median_is_zero {
311 format!(
313 " [{} – {}] {}",
314 min_val,
315 max_val,
316 spark(all_measurements.as_slice())
317 )
318 } else {
319 let relative_min = min_val / tail_median - 1.0;
322 let relative_max = max_val / tail_median - 1.0;
323
324 format!(
325 " [{:+.2}% – {:+.2}%] {}",
326 (relative_min * 100.0),
327 (relative_max * 100.0),
328 spark(all_measurements.as_slice())
329 )
330 };
331
332 let build_summary = || -> String {
335 let mut summary = String::new();
336
337 let total_measurements = all_measurements.len();
339
340 if total_measurements == 1 {
342 let head_display = StatsWithUnit {
343 stats: &head_summary,
344 unit: unit_str,
345 };
346 summary.push_str(&format!("Head: {}\n", head_display));
347 } else if total_measurements >= 2 {
348 let direction = get_direction_arrow(head_summary.mean, tail_summary.mean);
350 let z_score = head_summary.z_score_with_method(&tail_summary, dispersion_method);
351 let z_score_display = format_z_score_display(z_score);
352 let method_name = match dispersion_method {
353 DispersionMethod::StandardDeviation => "stddev",
354 DispersionMethod::MedianAbsoluteDeviation => "mad",
355 };
356
357 let head_display = StatsWithUnit {
358 stats: &head_summary,
359 unit: unit_str,
360 };
361 let tail_display = StatsWithUnit {
362 stats: &tail_summary,
363 unit: unit_str,
364 };
365
366 summary.push_str(&format!(
367 "z-score ({method_name}): {direction}{}\n",
368 z_score_display
369 ));
370 summary.push_str(&format!("Head: {}\n", head_display));
371 summary.push_str(&format!("Tail: {}\n", tail_display));
372 summary.push_str(&sparkline);
373 }
374 summary
377 };
378
379 if tail_summary.len < min_count.into() {
381 let number_measurements = tail_summary.len;
382 let plural_s = if number_measurements > 1 { "s" } else { "" };
384 error!("Only {number_measurements} measurement{plural_s} found. Less than requested min_measurements of {min_count}. Skipping test.");
385
386 let mut skip_message = format!(
387 "⏭️ '{measurement}'\nOnly {number_measurements} measurement{plural_s} found. Less than requested min_measurements of {min_count}. Skipping test."
388 );
389
390 let summary = build_summary();
392 if !summary.is_empty() {
393 skip_message.push('\n');
394 skip_message.push_str(&summary);
395 }
396
397 return Ok(AuditResult {
398 message: skip_message,
399 passed: true,
400 });
401 }
402
403 let head_relative_deviation = (head / tail_median - 1.0).abs() * 100.0;
406
407 let min_relative_deviation = config::audit_min_relative_deviation(measurement);
409 let threshold_applied = min_relative_deviation.is_some();
410
411 let passed_due_to_threshold = min_relative_deviation
413 .map(|threshold| head_relative_deviation < threshold)
414 .unwrap_or(false);
415
416 let text_summary = build_summary();
417
418 let z_score_exceeds_sigma =
420 head_summary.is_significant(&tail_summary, sigma, dispersion_method);
421
422 let passed = !z_score_exceeds_sigma || passed_due_to_threshold;
424
425 let threshold_note = if threshold_applied && passed_due_to_threshold && z_score_exceeds_sigma {
428 format!(
429 "\nNote: Passed due to relative deviation ({:.1}%) being below threshold ({:.1}%)",
430 head_relative_deviation,
431 min_relative_deviation.unwrap()
432 )
433 } else {
434 String::new()
435 };
436
437 if !passed {
439 return Ok(AuditResult {
440 message: format!(
441 "❌ '{measurement}'\nHEAD differs significantly from tail measurements.\n{text_summary}{threshold_note}"
442 ),
443 passed: false,
444 });
445 }
446
447 Ok(AuditResult {
448 message: format!("✅ '{measurement}'\n{text_summary}{threshold_note}"),
449 passed: true,
450 })
451}
452
453#[cfg(test)]
454mod test {
455 use super::*;
456
457 #[test]
458 fn test_format_z_score_display() {
459 let test_cases = vec![
461 (2.5_f64, " 2.50"),
462 (0.0_f64, " 0.00"),
463 (-1.5_f64, " -1.50"),
464 (999.999_f64, " 1000.00"),
465 (0.001_f64, " 0.00"),
466 (f64::INFINITY, ""),
467 (f64::NEG_INFINITY, ""),
468 (f64::NAN, ""),
469 ];
470
471 for (z_score, expected) in test_cases {
472 let result = format_z_score_display(z_score);
473 assert_eq!(result, expected, "Failed for z_score: {}", z_score);
474 }
475 }
476
477 #[test]
478 fn test_direction_arrows() {
479 let test_cases = vec![
481 (5.0_f64, 3.0_f64, "↑"), (1.0_f64, 3.0_f64, "↓"), (3.0_f64, 3.0_f64, "→"), ];
485
486 for (head_mean, tail_mean, expected) in test_cases {
487 let result = get_direction_arrow(head_mean, tail_mean);
488 assert_eq!(
489 result, expected,
490 "Failed for head_mean: {}, tail_mean: {}",
491 head_mean, tail_mean
492 );
493 }
494 }
495
496 #[test]
497 fn test_audit_with_different_dispersion_methods() {
498 let head_value = 35.0;
502 let tail_values = [30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 100.0];
503
504 let head_summary = stats::aggregate_measurements(std::iter::once(&head_value));
505 let tail_summary = stats::aggregate_measurements(tail_values.iter());
506
507 let z_score_stddev =
509 head_summary.z_score_with_method(&tail_summary, DispersionMethod::StandardDeviation);
510 let z_score_mad = head_summary
511 .z_score_with_method(&tail_summary, DispersionMethod::MedianAbsoluteDeviation);
512
513 assert!(
516 z_score_stddev < z_score_mad,
517 "stddev z-score ({}) should be smaller than MAD z-score ({}) with outlier data",
518 z_score_stddev,
519 z_score_mad
520 );
521
522 assert!(z_score_stddev > 0.0);
524 assert!(z_score_mad > 0.0);
525 }
526
527 #[test]
528 fn test_dispersion_method_conversion() {
529 let cli_stddev = git_perf_cli_types::DispersionMethod::StandardDeviation;
533 let stats_stddev: DispersionMethod = cli_stddev.into();
534 assert_eq!(stats_stddev, DispersionMethod::StandardDeviation);
535
536 let cli_mad = git_perf_cli_types::DispersionMethod::MedianAbsoluteDeviation;
538 let stats_mad: DispersionMethod = cli_mad.into();
539 assert_eq!(stats_mad, DispersionMethod::MedianAbsoluteDeviation);
540 }
541
542 #[test]
543 fn test_audit_multiple_with_no_measurements() {
544 let result = audit_multiple(
548 100,
549 Some(1),
550 &[],
551 Some(ReductionFunc::Mean),
552 Some(2.0),
553 Some(DispersionMethod::StandardDeviation),
554 &[], );
556
557 assert!(
559 result.is_ok(),
560 "audit_multiple should succeed with empty pattern list"
561 );
562 }
563
564 #[test]
567 fn test_min_count_boundary_condition() {
568 let result = audit_with_data(
571 "test_measurement",
572 15.0,
573 vec![10.0, 11.0, 12.0], 3, 2.0,
576 DispersionMethod::StandardDeviation,
577 );
578
579 assert!(result.is_ok());
580 let audit_result = result.unwrap();
581 assert!(!audit_result.message.contains("Skipping test"));
583
584 let result = audit_with_data(
586 "test_measurement",
587 15.0,
588 vec![10.0, 11.0], 3, 2.0,
591 DispersionMethod::StandardDeviation,
592 );
593
594 assert!(result.is_ok());
595 let audit_result = result.unwrap();
596 assert!(audit_result.message.contains("Skipping test"));
597 assert!(audit_result.passed); }
599
600 #[test]
601 fn test_pluralization_logic() {
602 let result = audit_with_data(
605 "test_measurement",
606 15.0,
607 vec![], 5, 2.0,
610 DispersionMethod::StandardDeviation,
611 );
612
613 assert!(result.is_ok());
614 let message = result.unwrap().message;
615 assert!(message.contains("0 measurement found")); assert!(!message.contains("0 measurements found")); let result = audit_with_data(
620 "test_measurement",
621 15.0,
622 vec![10.0], 5, 2.0,
625 DispersionMethod::StandardDeviation,
626 );
627
628 assert!(result.is_ok());
629 let message = result.unwrap().message;
630 assert!(message.contains("1 measurement found")); let result = audit_with_data(
634 "test_measurement",
635 15.0,
636 vec![10.0, 11.0], 5, 2.0,
639 DispersionMethod::StandardDeviation,
640 );
641
642 assert!(result.is_ok());
643 let message = result.unwrap().message;
644 assert!(message.contains("2 measurements found")); }
646
647 #[test]
648 fn test_skip_with_summaries() {
649 let result = audit_with_data(
655 "test_measurement",
656 15.0,
657 vec![], 5, 2.0,
660 DispersionMethod::StandardDeviation,
661 );
662
663 assert!(result.is_ok());
664 let message = result.unwrap().message;
665 assert!(message.contains("Skipping test"));
666 assert!(message.contains("Head:")); assert!(!message.contains("z-score")); assert!(!message.contains("Tail:")); assert!(!message.contains("[")); let result = audit_with_data(
673 "test_measurement",
674 15.0,
675 vec![10.0], 5, 2.0,
678 DispersionMethod::StandardDeviation,
679 );
680
681 assert!(result.is_ok());
682 let message = result.unwrap().message;
683 assert!(message.contains("Skipping test"));
684 assert!(message.contains("z-score (stddev):")); assert!(message.contains("Head:")); assert!(message.contains("Tail:")); assert!(message.contains("[")); let z_pos = message.find("z-score").unwrap();
690 let head_pos = message.find("Head:").unwrap();
691 let tail_pos = message.find("Tail:").unwrap();
692 let spark_pos = message.find("[").unwrap();
693 assert!(z_pos < head_pos, "z-score should come before Head");
694 assert!(head_pos < tail_pos, "Head should come before Tail");
695 assert!(tail_pos < spark_pos, "Tail should come before sparkline");
696
697 let result = audit_with_data(
699 "test_measurement",
700 15.0,
701 vec![10.0, 11.0], 5, 2.0,
704 DispersionMethod::StandardDeviation,
705 );
706
707 assert!(result.is_ok());
708 let message = result.unwrap().message;
709 assert!(message.contains("Skipping test"));
710 assert!(message.contains("z-score (stddev):")); assert!(message.contains("Head:")); assert!(message.contains("Tail:")); assert!(message.contains("[")); let z_pos = message.find("z-score").unwrap();
716 let head_pos = message.find("Head:").unwrap();
717 let tail_pos = message.find("Tail:").unwrap();
718 let spark_pos = message.find("[").unwrap();
719 assert!(z_pos < head_pos, "z-score should come before Head");
720 assert!(head_pos < tail_pos, "Head should come before Tail");
721 assert!(tail_pos < spark_pos, "Tail should come before sparkline");
722
723 let result = audit_with_data(
725 "test_measurement",
726 15.0,
727 vec![10.0, 11.0], 5, 2.0,
730 DispersionMethod::MedianAbsoluteDeviation,
731 );
732
733 assert!(result.is_ok());
734 let message = result.unwrap().message;
735 assert!(message.contains("z-score (mad):")); }
737
738 #[test]
739 fn test_relative_calculations_division_vs_modulo() {
740 let result = audit_with_data(
743 "test_measurement",
744 25.0, vec![10.0, 10.0, 10.0], 1,
747 10.0, DispersionMethod::StandardDeviation,
749 );
750
751 assert!(result.is_ok());
752 let audit_result = result.unwrap();
753
754 assert!(audit_result.message.contains("[+0.00% – +150.00%]"));
764
765 assert!(!audit_result.message.contains("[-100.00% – -50.00%]"));
767 assert!(!audit_result.message.contains("-100.00%"));
768 assert!(!audit_result.message.contains("-50.00%"));
769 }
770
771 #[test]
772 fn test_core_pass_fail_logic() {
773 let result = audit_with_data(
778 "test_measurement", 100.0, vec![10.0, 10.0, 10.0, 10.0, 10.0], 1,
782 0.5, DispersionMethod::StandardDeviation,
784 );
785
786 assert!(result.is_ok());
787 let audit_result = result.unwrap();
788 assert!(!audit_result.passed); assert!(audit_result.message.contains("❌"));
790
791 let result = audit_with_data(
793 "test_measurement",
794 10.2, vec![10.0, 10.1, 10.0, 10.1, 10.0], 1,
797 100.0, DispersionMethod::StandardDeviation,
799 );
800
801 assert!(result.is_ok());
802 let audit_result = result.unwrap();
803 assert!(audit_result.passed); assert!(audit_result.message.contains("✅"));
805 }
806
807 #[test]
808 fn test_final_result_logic() {
809 let result = audit_with_data(
814 "test_measurement",
815 1000.0, vec![10.0, 10.0, 10.0, 10.0, 10.0],
817 1,
818 0.1, DispersionMethod::StandardDeviation,
820 );
821
822 assert!(result.is_ok());
823 let audit_result = result.unwrap();
824 assert!(!audit_result.passed);
825 assert!(audit_result.message.contains("❌"));
826 assert!(audit_result.message.contains("differs significantly"));
827
828 let result = audit_with_data(
830 "test_measurement",
831 10.01, vec![10.0, 10.1, 10.0, 10.1, 10.0], 1,
834 100.0, DispersionMethod::StandardDeviation,
836 );
837
838 assert!(result.is_ok());
839 let audit_result = result.unwrap();
840 assert!(audit_result.passed);
841 assert!(audit_result.message.contains("✅"));
842 assert!(!audit_result.message.contains("differs significantly"));
843 }
844
845 #[test]
846 fn test_dispersion_methods_produce_different_results() {
847 let head = 35.0;
849 let tail = vec![30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 100.0];
850
851 let result_stddev = audit_with_data(
852 "test_measurement",
853 head,
854 tail.clone(),
855 1,
856 2.0,
857 DispersionMethod::StandardDeviation,
858 );
859
860 let result_mad = audit_with_data(
861 "test_measurement",
862 head,
863 tail,
864 1,
865 2.0,
866 DispersionMethod::MedianAbsoluteDeviation,
867 );
868
869 assert!(result_stddev.is_ok());
870 assert!(result_mad.is_ok());
871
872 let stddev_result = result_stddev.unwrap();
873 let mad_result = result_mad.unwrap();
874
875 assert!(stddev_result.message.contains("stddev"));
877 assert!(mad_result.message.contains("mad"));
878 }
879
880 #[test]
881 fn test_head_and_tail_have_units_and_auto_scaling() {
882 use crate::test_helpers::setup_test_env_with_config;
886 use std::env;
887
888 let config_content = r#"
889[measurement."build_time"]
890unit = "ms"
891"#;
892 let (_temp_dir, _dir_guard) = setup_test_env_with_config(config_content);
893
894 let head = 12_345.67; let tail = vec![10_000.0, 10_500.0, 11_000.0, 11_500.0, 12_000.0]; let result = audit_with_data(
899 "build_time",
900 head,
901 tail,
902 1,
903 10.0, DispersionMethod::StandardDeviation,
905 );
906
907 assert!(result.is_ok());
908 let audit_result = result.unwrap();
909 let message = &audit_result.message;
910
911 assert!(
913 message.contains("Head:"),
914 "Message should contain Head section"
915 );
916
917 assert!(
920 message.contains("12.3s") || message.contains("12.35s"),
921 "Head mean should be auto-scaled to seconds, got: {}",
922 message
923 );
924
925 let head_section: Vec<&str> = message
926 .lines()
927 .filter(|line| line.contains("Head:"))
928 .collect();
929
930 assert!(
931 !head_section.is_empty(),
932 "Should find Head section in message"
933 );
934
935 let head_line = head_section[0];
936
937 assert!(
940 head_line.contains("μ:") && head_line.contains("σ:") && head_line.contains("MAD:"),
941 "Head line should contain μ, σ, and MAD labels, got: {}",
942 head_line
943 );
944
945 assert!(
947 message.contains("Tail:"),
948 "Message should contain Tail section"
949 );
950
951 let tail_section: Vec<&str> = message
952 .lines()
953 .filter(|line| line.contains("Tail:"))
954 .collect();
955
956 assert!(
957 !tail_section.is_empty(),
958 "Should find Tail section in message"
959 );
960
961 let tail_line = tail_section[0];
962
963 assert!(
965 tail_line.contains("11s")
966 || tail_line.contains("11.")
967 || tail_line.contains("10.")
968 || tail_line.contains("12."),
969 "Tail should contain auto-scaled second values, got: {}",
970 tail_line
971 );
972
973 assert!(
975 tail_line.contains("μ:")
976 && tail_line.contains("σ:")
977 && tail_line.contains("MAD:")
978 && tail_line.contains("n:"),
979 "Tail line should contain all stat labels, got: {}",
980 tail_line
981 );
982 }
983
984 #[test]
985 fn test_threshold_note_only_shown_when_audit_would_fail() {
986 use crate::test_helpers::setup_test_env_with_config;
989
990 let config_content = r#"
991[measurement."build_time"]
992min_relative_deviation = 10.0
993"#;
994 let (_temp_dir, _dir_guard) = setup_test_env_with_config(config_content);
995
996 let result = audit_with_data(
999 "build_time",
1000 10.1, vec![10.0, 10.1, 10.0, 10.1, 10.0], 1,
1003 100.0, DispersionMethod::StandardDeviation,
1005 );
1006
1007 assert!(result.is_ok());
1008 let audit_result = result.unwrap();
1009 assert!(audit_result.passed);
1010 assert!(audit_result.message.contains("✅"));
1011 assert!(
1013 !audit_result
1014 .message
1015 .contains("Note: Passed due to relative deviation"),
1016 "Note should not appear when audit passes without needing threshold bypass"
1017 );
1018
1019 let result = audit_with_data(
1022 "build_time",
1023 1002.0, vec![1000.0, 1000.1, 1000.0, 1000.1, 1000.0], 1,
1026 0.5, DispersionMethod::StandardDeviation,
1028 );
1029
1030 assert!(result.is_ok());
1031 let audit_result = result.unwrap();
1032 assert!(audit_result.passed);
1033 assert!(audit_result.message.contains("✅"));
1034 assert!(
1036 audit_result
1037 .message
1038 .contains("Note: Passed due to relative deviation"),
1039 "Note should appear when audit passes due to threshold bypass. Got: {}",
1040 audit_result.message
1041 );
1042
1043 let result = audit_with_data(
1046 "build_time",
1047 1200.0, vec![1000.0, 1000.1, 1000.0, 1000.1, 1000.0], 1,
1050 0.5, DispersionMethod::StandardDeviation,
1052 );
1053
1054 assert!(result.is_ok());
1055 let audit_result = result.unwrap();
1056 assert!(!audit_result.passed);
1057 assert!(audit_result.message.contains("❌"));
1058 assert!(
1060 !audit_result
1061 .message
1062 .contains("Note: Passed due to relative deviation"),
1063 "Note should not appear when audit fails"
1064 );
1065 }
1066
1067 #[cfg(test)]
1069 mod integration {
1070 use super::*;
1071 use crate::config::{
1072 audit_aggregate_by, audit_dispersion_method, audit_min_measurements, audit_sigma,
1073 };
1074 use crate::test_helpers::setup_test_env_with_config;
1075 use std::env;
1076
1077 #[test]
1078 fn test_different_dispersion_methods_per_measurement() {
1079 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1080 r#"
1081[measurement]
1082dispersion_method = "stddev"
1083
1084[measurement."build_time"]
1085dispersion_method = "mad"
1086
1087[measurement."memory_usage"]
1088dispersion_method = "stddev"
1089"#,
1090 );
1091
1092 let build_time_method = audit_dispersion_method("build_time");
1094 let memory_usage_method = audit_dispersion_method("memory_usage");
1095 let other_method = audit_dispersion_method("other_metric");
1096
1097 assert_eq!(
1098 DispersionMethod::from(build_time_method),
1099 DispersionMethod::MedianAbsoluteDeviation,
1100 "build_time should use MAD"
1101 );
1102 assert_eq!(
1103 DispersionMethod::from(memory_usage_method),
1104 DispersionMethod::StandardDeviation,
1105 "memory_usage should use stddev"
1106 );
1107 assert_eq!(
1108 DispersionMethod::from(other_method),
1109 DispersionMethod::StandardDeviation,
1110 "other_metric should use default stddev"
1111 );
1112 }
1113
1114 #[test]
1115 fn test_different_min_measurements_per_measurement() {
1116 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1117 r#"
1118[measurement]
1119min_measurements = 5
1120
1121[measurement."build_time"]
1122min_measurements = 10
1123
1124[measurement."memory_usage"]
1125min_measurements = 3
1126"#,
1127 );
1128
1129 assert_eq!(
1130 audit_min_measurements("build_time"),
1131 Some(10),
1132 "build_time should require 10 measurements"
1133 );
1134 assert_eq!(
1135 audit_min_measurements("memory_usage"),
1136 Some(3),
1137 "memory_usage should require 3 measurements"
1138 );
1139 assert_eq!(
1140 audit_min_measurements("other_metric"),
1141 Some(5),
1142 "other_metric should use default 5 measurements"
1143 );
1144 }
1145
1146 #[test]
1147 fn test_different_aggregate_by_per_measurement() {
1148 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1149 r#"
1150[measurement]
1151aggregate_by = "median"
1152
1153[measurement."build_time"]
1154aggregate_by = "max"
1155
1156[measurement."memory_usage"]
1157aggregate_by = "mean"
1158"#,
1159 );
1160
1161 assert_eq!(
1162 audit_aggregate_by("build_time"),
1163 Some(git_perf_cli_types::ReductionFunc::Max),
1164 "build_time should use max"
1165 );
1166 assert_eq!(
1167 audit_aggregate_by("memory_usage"),
1168 Some(git_perf_cli_types::ReductionFunc::Mean),
1169 "memory_usage should use mean"
1170 );
1171 assert_eq!(
1172 audit_aggregate_by("other_metric"),
1173 Some(git_perf_cli_types::ReductionFunc::Median),
1174 "other_metric should use default median"
1175 );
1176 }
1177
1178 #[test]
1179 fn test_different_sigma_per_measurement() {
1180 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1181 r#"
1182[measurement]
1183sigma = 3.0
1184
1185[measurement."build_time"]
1186sigma = 5.5
1187
1188[measurement."memory_usage"]
1189sigma = 2.0
1190"#,
1191 );
1192
1193 assert_eq!(
1194 audit_sigma("build_time"),
1195 Some(5.5),
1196 "build_time should use sigma 5.5"
1197 );
1198 assert_eq!(
1199 audit_sigma("memory_usage"),
1200 Some(2.0),
1201 "memory_usage should use sigma 2.0"
1202 );
1203 assert_eq!(
1204 audit_sigma("other_metric"),
1205 Some(3.0),
1206 "other_metric should use default sigma 3.0"
1207 );
1208 }
1209
1210 #[test]
1211 fn test_cli_overrides_config() {
1212 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1213 r#"
1214[measurement."build_time"]
1215min_measurements = 10
1216aggregate_by = "max"
1217sigma = 5.5
1218dispersion_method = "mad"
1219"#,
1220 );
1221
1222 let params = super::resolve_audit_params(
1224 "build_time",
1225 Some(2), Some(ReductionFunc::Min), Some(3.0), Some(DispersionMethod::StandardDeviation), );
1230
1231 assert_eq!(
1232 params.min_count, 2,
1233 "CLI min_measurements should override config"
1234 );
1235 assert_eq!(
1236 params.summarize_by,
1237 ReductionFunc::Min,
1238 "CLI aggregate_by should override config"
1239 );
1240 assert_eq!(params.sigma, 3.0, "CLI sigma should override config");
1241 assert_eq!(
1242 params.dispersion_method,
1243 DispersionMethod::StandardDeviation,
1244 "CLI dispersion should override config"
1245 );
1246 }
1247
1248 #[test]
1249 fn test_config_overrides_defaults() {
1250 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1251 r#"
1252[measurement."build_time"]
1253min_measurements = 10
1254aggregate_by = "max"
1255sigma = 5.5
1256dispersion_method = "mad"
1257"#,
1258 );
1259
1260 let params = super::resolve_audit_params(
1262 "build_time",
1263 None, None,
1265 None,
1266 None,
1267 );
1268
1269 assert_eq!(
1270 params.min_count, 10,
1271 "Config min_measurements should override default"
1272 );
1273 assert_eq!(
1274 params.summarize_by,
1275 ReductionFunc::Max,
1276 "Config aggregate_by should override default"
1277 );
1278 assert_eq!(params.sigma, 5.5, "Config sigma should override default");
1279 assert_eq!(
1280 params.dispersion_method,
1281 DispersionMethod::MedianAbsoluteDeviation,
1282 "Config dispersion should override default"
1283 );
1284 }
1285
1286 #[test]
1287 fn test_uses_defaults_when_no_config_or_cli() {
1288 let (_temp_dir, _dir_guard) = setup_test_env_with_config("");
1289
1290 let params = super::resolve_audit_params(
1292 "non_existent_measurement",
1293 None, None,
1295 None,
1296 None,
1297 );
1298
1299 assert_eq!(
1300 params.min_count, 2,
1301 "Should use default min_measurements of 2"
1302 );
1303 assert_eq!(
1304 params.summarize_by,
1305 ReductionFunc::Min,
1306 "Should use default aggregate_by of Min"
1307 );
1308 assert_eq!(params.sigma, 4.0, "Should use default sigma of 4.0");
1309 assert_eq!(
1310 params.dispersion_method,
1311 DispersionMethod::StandardDeviation,
1312 "Should use default dispersion of stddev"
1313 );
1314 }
1315 }
1316
1317 #[test]
1318 fn test_discover_matching_measurements() {
1319 use crate::data::{Commit, MeasurementData};
1320 use std::collections::HashMap;
1321
1322 let commits = vec![
1324 Ok(Commit {
1325 commit: "abc123".to_string(),
1326 measurements: vec![
1327 MeasurementData {
1328 epoch: 0,
1329 name: "bench_cpu".to_string(),
1330 timestamp: 1000.0,
1331 val: 100.0,
1332 key_values: {
1333 let mut map = HashMap::new();
1334 map.insert("os".to_string(), "linux".to_string());
1335 map
1336 },
1337 },
1338 MeasurementData {
1339 epoch: 0,
1340 name: "bench_memory".to_string(),
1341 timestamp: 1000.0,
1342 val: 200.0,
1343 key_values: {
1344 let mut map = HashMap::new();
1345 map.insert("os".to_string(), "linux".to_string());
1346 map
1347 },
1348 },
1349 MeasurementData {
1350 epoch: 0,
1351 name: "test_unit".to_string(),
1352 timestamp: 1000.0,
1353 val: 50.0,
1354 key_values: {
1355 let mut map = HashMap::new();
1356 map.insert("os".to_string(), "linux".to_string());
1357 map
1358 },
1359 },
1360 ],
1361 }),
1362 Ok(Commit {
1363 commit: "def456".to_string(),
1364 measurements: vec![
1365 MeasurementData {
1366 epoch: 0,
1367 name: "bench_cpu".to_string(),
1368 timestamp: 1000.0,
1369 val: 105.0,
1370 key_values: {
1371 let mut map = HashMap::new();
1372 map.insert("os".to_string(), "mac".to_string());
1373 map
1374 },
1375 },
1376 MeasurementData {
1377 epoch: 0,
1378 name: "other_metric".to_string(),
1379 timestamp: 1000.0,
1380 val: 75.0,
1381 key_values: {
1382 let mut map = HashMap::new();
1383 map.insert("os".to_string(), "linux".to_string());
1384 map
1385 },
1386 },
1387 ],
1388 }),
1389 ];
1390
1391 let patterns = vec!["bench_.*".to_string()];
1393 let filters = crate::filter::compile_filters(&patterns).unwrap();
1394 let selectors = vec![];
1395 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1396
1397 assert_eq!(discovered.len(), 2);
1398 assert!(discovered.contains(&"bench_cpu".to_string()));
1399 assert!(discovered.contains(&"bench_memory".to_string()));
1400 assert!(!discovered.contains(&"test_unit".to_string()));
1401 assert!(!discovered.contains(&"other_metric".to_string()));
1402
1403 let patterns = vec!["bench_cpu".to_string(), "test_.*".to_string()];
1405 let filters = crate::filter::compile_filters(&patterns).unwrap();
1406 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1407
1408 assert_eq!(discovered.len(), 2);
1409 assert!(discovered.contains(&"bench_cpu".to_string()));
1410 assert!(discovered.contains(&"test_unit".to_string()));
1411 assert!(!discovered.contains(&"bench_memory".to_string()));
1412
1413 let patterns = vec!["bench_.*".to_string()];
1415 let filters = crate::filter::compile_filters(&patterns).unwrap();
1416 let selectors = vec![("os".to_string(), "linux".to_string())];
1417 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1418
1419 assert_eq!(discovered.len(), 2);
1422 assert!(discovered.contains(&"bench_cpu".to_string()));
1423 assert!(discovered.contains(&"bench_memory".to_string()));
1424
1425 let patterns = vec!["nonexistent.*".to_string()];
1427 let filters = crate::filter::compile_filters(&patterns).unwrap();
1428 let selectors = vec![];
1429 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1430
1431 assert_eq!(discovered.len(), 0);
1432
1433 let filters = vec![];
1435 let selectors = vec![];
1436 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1437
1438 assert_eq!(discovered.len(), 4);
1442 assert!(discovered.contains(&"bench_cpu".to_string()));
1443 assert!(discovered.contains(&"bench_memory".to_string()));
1444 assert!(discovered.contains(&"test_unit".to_string()));
1445 assert!(discovered.contains(&"other_metric".to_string()));
1446
1447 let patterns = vec!["bench_.*".to_string()];
1449 let filters = crate::filter::compile_filters(&patterns).unwrap();
1450 let selectors = vec![("os".to_string(), "windows".to_string())];
1451 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1452
1453 assert_eq!(discovered.len(), 0);
1454
1455 let patterns = vec!["^bench_cpu$".to_string()];
1457 let filters = crate::filter::compile_filters(&patterns).unwrap();
1458 let selectors = vec![];
1459 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1460
1461 assert_eq!(discovered.len(), 1);
1462 assert!(discovered.contains(&"bench_cpu".to_string()));
1463
1464 let patterns = vec![".*".to_string()]; let filters = crate::filter::compile_filters(&patterns).unwrap();
1467 let selectors = vec![];
1468 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1469
1470 assert_eq!(discovered[0], "bench_cpu");
1472 assert_eq!(discovered[1], "bench_memory");
1473 assert_eq!(discovered[2], "other_metric");
1474 assert_eq!(discovered[3], "test_unit");
1475 }
1476
1477 #[test]
1478 fn test_audit_multiple_with_combined_patterns() {
1479 use crate::data::{Commit, MeasurementData};
1486 use std::collections::HashMap;
1487
1488 let commits = vec![Ok(Commit {
1490 commit: "abc123".to_string(),
1491 measurements: vec![
1492 MeasurementData {
1493 epoch: 0,
1494 name: "timer".to_string(),
1495 timestamp: 1000.0,
1496 val: 10.0,
1497 key_values: HashMap::new(),
1498 },
1499 MeasurementData {
1500 epoch: 0,
1501 name: "bench_cpu".to_string(),
1502 timestamp: 1000.0,
1503 val: 100.0,
1504 key_values: HashMap::new(),
1505 },
1506 MeasurementData {
1507 epoch: 0,
1508 name: "memory".to_string(),
1509 timestamp: 1000.0,
1510 val: 500.0,
1511 key_values: HashMap::new(),
1512 },
1513 ],
1514 })];
1515
1516 let measurements = vec!["timer".to_string()];
1519 let filter_patterns = vec!["bench_.*".to_string()];
1520 let combined =
1521 crate::filter::combine_measurements_and_filters(&measurements, &filter_patterns);
1522
1523 assert_eq!(combined.len(), 2);
1525 assert_eq!(combined[0], "^timer$");
1526 assert_eq!(combined[1], "bench_.*");
1527
1528 let filters = crate::filter::compile_filters(&combined).unwrap();
1530 let selectors = vec![];
1531 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1532
1533 assert_eq!(discovered.len(), 2);
1535 assert!(discovered.contains(&"timer".to_string()));
1536 assert!(discovered.contains(&"bench_cpu".to_string()));
1537 assert!(!discovered.contains(&"memory".to_string())); let measurements = vec!["timer".to_string(), "memory".to_string()];
1541 let filter_patterns = vec!["bench_.*".to_string(), "test_.*".to_string()];
1542 let combined =
1543 crate::filter::combine_measurements_and_filters(&measurements, &filter_patterns);
1544
1545 assert_eq!(combined.len(), 4);
1546
1547 let filters = crate::filter::compile_filters(&combined).unwrap();
1548 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1549
1550 assert_eq!(discovered.len(), 3);
1552 assert!(discovered.contains(&"timer".to_string()));
1553 assert!(discovered.contains(&"memory".to_string()));
1554 assert!(discovered.contains(&"bench_cpu".to_string()));
1555 }
1556
1557 #[test]
1558 fn test_audit_with_empty_tail() {
1559 let result = audit_with_data(
1563 "test_measurement",
1564 10.0, vec![], 2, 2.0, DispersionMethod::StandardDeviation,
1569 );
1570
1571 assert!(result.is_ok(), "Should not crash on empty tail");
1573 let audit_result = result.unwrap();
1574
1575 assert!(audit_result.passed);
1577 assert!(audit_result.message.contains("Skipping test"));
1578
1579 assert!(!audit_result.message.to_lowercase().contains("inf"));
1581 assert!(!audit_result.message.to_lowercase().contains("nan"));
1582 }
1583
1584 #[test]
1585 fn test_audit_with_all_zero_tail() {
1586 let result = audit_with_data(
1589 "test_measurement",
1590 5.0, vec![0.0, 0.0, 0.0], 2, 2.0, DispersionMethod::StandardDeviation,
1595 );
1596
1597 assert!(result.is_ok(), "Should not crash when tail median is 0.0");
1599 let audit_result = result.unwrap();
1600
1601 assert!(!audit_result.message.to_lowercase().contains("inf"));
1603 assert!(!audit_result.message.to_lowercase().contains("nan"));
1604 }
1605
1606 #[test]
1607 fn test_tiered_baseline_approach() {
1608 let result = audit_with_data(
1614 "test_measurement",
1615 15.0, vec![10.0, 11.0, 12.0], 2,
1618 2.0,
1619 DispersionMethod::StandardDeviation,
1620 );
1621
1622 assert!(result.is_ok());
1623 let audit_result = result.unwrap();
1624 assert!(audit_result.message.contains('%'));
1626 assert!(!audit_result.message.to_lowercase().contains("inf"));
1627
1628 let result = audit_with_data(
1630 "test_measurement",
1631 5.0, vec![0.0, 0.0, 0.0], 2,
1634 2.0,
1635 DispersionMethod::StandardDeviation,
1636 );
1637
1638 assert!(result.is_ok());
1639 let audit_result = result.unwrap();
1640 assert!(!audit_result.message.to_lowercase().contains("inf"));
1643 assert!(!audit_result.message.to_lowercase().contains("nan"));
1644 assert!(audit_result.message.contains('–') || audit_result.message.contains('-'));
1646
1647 let result = audit_with_data(
1649 "test_measurement",
1650 0.0, vec![0.0, 0.0, 0.0], 2,
1653 2.0,
1654 DispersionMethod::StandardDeviation,
1655 );
1656
1657 assert!(result.is_ok());
1658 let audit_result = result.unwrap();
1659 assert!(!audit_result.message.to_lowercase().contains("inf"));
1661 assert!(!audit_result.message.to_lowercase().contains("nan"));
1662 }
1663}