1use crate::{
2 config,
3 data::{Commit, MeasurementData},
4 defaults,
5 measurement_retrieval::{self, summarize_measurements},
6 stats::{self, DispersionMethod, ReductionFunc, StatsWithUnit, VecAggregation},
7};
8use anyhow::{anyhow, bail, Result};
9use itertools::Itertools;
10use log::info;
11use sparklines::spark;
12use std::cmp::Ordering;
13use std::collections::HashSet;
14use std::iter;
15
16fn format_z_score_display(z_score: f64) -> String {
20 if z_score.is_finite() {
21 format!(" {:.2}", z_score)
22 } else {
23 String::new()
24 }
25}
26
27fn get_direction_arrow(head_mean: f64, tail_mean: f64) -> &'static str {
31 match head_mean.partial_cmp(&tail_mean) {
32 Some(Ordering::Greater) => "↑",
33 Some(Ordering::Less) => "↓",
34 Some(Ordering::Equal) | None => "→",
35 }
36}
37
38#[derive(Debug, PartialEq)]
39struct AuditResult {
40 message: String,
41 passed: bool,
42}
43
44#[derive(Debug, PartialEq)]
46pub(crate) struct ResolvedAuditParams {
47 pub min_count: u16,
48 pub summarize_by: ReductionFunc,
49 pub sigma: f64,
50 pub dispersion_method: DispersionMethod,
51}
52
53pub(crate) fn resolve_audit_params(
59 measurement: &str,
60 cli_min_count: Option<u16>,
61 cli_summarize_by: Option<ReductionFunc>,
62 cli_sigma: Option<f64>,
63 cli_dispersion_method: Option<DispersionMethod>,
64) -> ResolvedAuditParams {
65 let min_count = cli_min_count
66 .or_else(|| config::audit_min_measurements(measurement))
67 .unwrap_or(defaults::DEFAULT_MIN_MEASUREMENTS);
68
69 let summarize_by = cli_summarize_by
70 .or_else(|| config::audit_aggregate_by(measurement).map(ReductionFunc::from))
71 .unwrap_or(ReductionFunc::Min);
72
73 let sigma = cli_sigma
74 .or_else(|| config::audit_sigma(measurement))
75 .unwrap_or(defaults::DEFAULT_SIGMA);
76
77 let dispersion_method = cli_dispersion_method
78 .or_else(|| {
79 Some(DispersionMethod::from(config::audit_dispersion_method(
80 measurement,
81 )))
82 })
83 .unwrap_or(DispersionMethod::StandardDeviation);
84
85 ResolvedAuditParams {
86 min_count,
87 summarize_by,
88 sigma,
89 dispersion_method,
90 }
91}
92
93fn discover_matching_measurements(
96 commits: &[Result<Commit>],
97 filters: &[regex::Regex],
98 selectors: &[(String, String)],
99) -> Vec<String> {
100 let mut unique_measurements = HashSet::new();
101
102 for commit in commits.iter().flatten() {
103 for measurement in &commit.measurements {
104 if !crate::filter::matches_any_filter(&measurement.name, filters) {
106 continue;
107 }
108
109 if !measurement.key_values_is_superset_of(selectors) {
111 continue;
112 }
113
114 unique_measurements.insert(measurement.name.clone());
116 }
117 }
118
119 let mut result: Vec<String> = unique_measurements.into_iter().collect();
121 result.sort();
122 result
123}
124
125#[allow(clippy::too_many_arguments)]
126pub fn audit_multiple(
127 start_commit: &str,
128 max_count: usize,
129 min_count: Option<u16>,
130 selectors: &[(String, String)],
131 summarize_by: Option<ReductionFunc>,
132 sigma: Option<f64>,
133 dispersion_method: Option<DispersionMethod>,
134 combined_patterns: &[String],
135 _no_change_point_warning: bool, ) -> Result<()> {
137 if combined_patterns.is_empty() {
139 return Ok(());
140 }
141
142 let filters = crate::filter::compile_filters(combined_patterns)?;
145
146 let all_commits: Vec<Result<Commit>> =
149 measurement_retrieval::walk_commits_from(start_commit, max_count)?.collect();
150
151 let measurements_to_audit = discover_matching_measurements(&all_commits, &filters, selectors);
154
155 if measurements_to_audit.is_empty() {
157 if all_commits.is_empty() {
159 bail!("No commit at HEAD");
160 }
161 let has_any_measurements = all_commits.iter().any(|commit_result| {
163 if let Ok(commit) = commit_result {
164 !commit.measurements.is_empty()
165 } else {
166 false
167 }
168 });
169
170 if !has_any_measurements {
171 bail!("No measurement for HEAD");
173 }
174 bail!("No measurements found matching the provided patterns");
176 }
177
178 let mut failed = false;
179
180 for measurement in measurements_to_audit {
182 let params = resolve_audit_params(
183 &measurement,
184 min_count,
185 summarize_by,
186 sigma,
187 dispersion_method,
188 );
189
190 if (max_count as u16) < params.min_count {
192 eprintln!(
193 "⚠️ Warning: --max_count ({}) is less than min_measurements ({}) for measurement '{}'.",
194 max_count, params.min_count, measurement
195 );
196 eprintln!(
197 " This limits available historical data and may prevent achieving statistical significance."
198 );
199 }
200
201 let result = audit_with_commits(
202 &measurement,
203 &all_commits,
204 params.min_count,
205 selectors,
206 params.summarize_by,
207 params.sigma,
208 params.dispersion_method,
209 )?;
210
211 println!("{}", result.message);
220
221 if !result.passed {
222 failed = true;
223 }
224 }
225
226 if failed {
227 bail!("One or more measurements failed audit.");
228 }
229
230 Ok(())
231}
232
233fn audit_with_commits(
237 measurement: &str,
238 commits: &[Result<Commit>],
239 min_count: u16,
240 selectors: &[(String, String)],
241 summarize_by: ReductionFunc,
242 sigma: f64,
243 dispersion_method: DispersionMethod,
244) -> Result<AuditResult> {
245 let commits_iter = commits.iter().map(|r| match r {
248 Ok(commit) => Ok(Commit {
249 commit: commit.commit.clone(),
250 title: commit.title.clone(),
251 author: commit.author.clone(),
252 measurements: commit.measurements.clone(),
253 }),
254 Err(e) => Err(anyhow::anyhow!("{}", e)),
255 });
256
257 let filter_by =
259 |m: &MeasurementData| m.name == measurement && m.key_values_is_superset_of(selectors);
260
261 let mut aggregates = measurement_retrieval::take_while_same_epoch(summarize_measurements(
262 commits_iter,
263 &summarize_by,
264 &filter_by,
265 ));
266
267 let head = aggregates
268 .next()
269 .ok_or(anyhow!("No commit at HEAD"))
270 .and_then(|s| {
271 s.and_then(|cs| {
272 cs.measurement
273 .map(|m| m.val)
274 .ok_or(anyhow!("No measurement for HEAD."))
275 })
276 })?;
277
278 let tail: Vec<_> = aggregates
279 .filter_map_ok(|cs| cs.measurement.map(|m| m.val))
280 .try_collect()?;
281
282 audit_with_data(measurement, head, tail, min_count, sigma, dispersion_method)
283}
284
285fn audit_with_data(
288 measurement: &str,
289 head: f64,
290 tail: Vec<f64>,
291 min_count: u16,
292 sigma: f64,
293 dispersion_method: DispersionMethod,
294) -> Result<AuditResult> {
295 assert!(min_count >= 2, "min_count must be at least 2");
299
300 let unit = config::measurement_unit(measurement);
302 let unit_str = unit.as_deref();
303
304 let head_summary = stats::aggregate_measurements(iter::once(&head));
305 let tail_summary = stats::aggregate_measurements(tail.iter());
306
307 let all_measurements = tail.into_iter().chain(iter::once(head)).collect::<Vec<_>>();
309
310 let mut tail_measurements = all_measurements.clone();
311 tail_measurements.pop(); let tail_median = tail_measurements.median().unwrap_or_default();
313
314 let min_val = all_measurements
316 .iter()
317 .min_by(|a, b| a.partial_cmp(b).unwrap())
318 .unwrap();
319 let max_val = all_measurements
320 .iter()
321 .max_by(|a, b| a.partial_cmp(b).unwrap())
322 .unwrap();
323
324 let tail_median_is_zero = tail_median.abs() < f64::EPSILON;
328
329 let sparkline = if tail_median_is_zero {
330 format!(
332 " [{} – {}] {}",
333 min_val,
334 max_val,
335 spark(all_measurements.as_slice())
336 )
337 } else {
338 let relative_min = min_val / tail_median - 1.0;
341 let relative_max = max_val / tail_median - 1.0;
342
343 format!(
344 " [{:+.2}% – {:+.2}%] {}",
345 (relative_min * 100.0),
346 (relative_max * 100.0),
347 spark(all_measurements.as_slice())
348 )
349 };
350
351 let build_summary = || -> String {
354 let mut summary = String::new();
355
356 let total_measurements = all_measurements.len();
358
359 if total_measurements == 1 {
361 let head_display = StatsWithUnit {
362 stats: &head_summary,
363 unit: unit_str,
364 };
365 summary.push_str(&format!("Head: {}\n", head_display));
366 } else if total_measurements >= 2 {
367 let direction = get_direction_arrow(head_summary.mean, tail_summary.mean);
369 let z_score = head_summary.z_score_with_method(&tail_summary, dispersion_method);
370 let z_score_display = format_z_score_display(z_score);
371 let method_name = match dispersion_method {
372 DispersionMethod::StandardDeviation => "stddev",
373 DispersionMethod::MedianAbsoluteDeviation => "mad",
374 };
375
376 let head_display = StatsWithUnit {
377 stats: &head_summary,
378 unit: unit_str,
379 };
380 let tail_display = StatsWithUnit {
381 stats: &tail_summary,
382 unit: unit_str,
383 };
384
385 summary.push_str(&format!(
386 "z-score ({method_name}): {direction}{}\n",
387 z_score_display
388 ));
389 summary.push_str(&format!("Head: {}\n", head_display));
390 summary.push_str(&format!("Tail: {}\n", tail_display));
391 summary.push_str(&sparkline);
392 }
393 summary
396 };
397
398 if tail_summary.len < min_count.into() {
400 let number_measurements = tail_summary.len;
401 let plural_s = if number_measurements == 1 { "" } else { "s" };
403 info!("Only {number_measurements} historical measurement{plural_s} found. Less than requested min_measurements of {min_count}. Skipping test.");
404
405 let mut skip_message = format!(
406 "⏭️ '{measurement}'\nOnly {number_measurements} historical measurement{plural_s} found. Less than requested min_measurements of {min_count}. Skipping test."
407 );
408
409 let summary = build_summary();
411 if !summary.is_empty() {
412 skip_message.push('\n');
413 skip_message.push_str(&summary);
414 }
415
416 return Ok(AuditResult {
417 message: skip_message,
418 passed: true,
419 });
420 }
421
422 let head_relative_deviation = (head / tail_median - 1.0).abs() * 100.0;
425
426 let min_relative_deviation = config::audit_min_relative_deviation(measurement);
428 let threshold_applied = min_relative_deviation.is_some();
429
430 let passed_due_to_threshold = min_relative_deviation
432 .map(|threshold| head_relative_deviation < threshold)
433 .unwrap_or(false);
434
435 let text_summary = build_summary();
436
437 let z_score_exceeds_sigma =
439 head_summary.is_significant(&tail_summary, sigma, dispersion_method);
440
441 let passed = !z_score_exceeds_sigma || passed_due_to_threshold;
443
444 let threshold_note = if threshold_applied && passed_due_to_threshold && z_score_exceeds_sigma {
447 format!(
448 "\nNote: Passed due to relative deviation ({:.1}%) being below threshold ({:.1}%)",
449 head_relative_deviation,
450 min_relative_deviation.unwrap()
451 )
452 } else {
453 String::new()
454 };
455
456 if !passed {
458 return Ok(AuditResult {
459 message: format!(
460 "❌ '{measurement}'\nHEAD differs significantly from tail measurements.\n{text_summary}{threshold_note}"
461 ),
462 passed: false,
463 });
464 }
465
466 Ok(AuditResult {
467 message: format!("✅ '{measurement}'\n{text_summary}{threshold_note}"),
468 passed: true,
469 })
470}
471
472#[cfg(test)]
473mod test {
474 use crate::test_helpers::with_isolated_test_setup;
475
476 use super::*;
477
478 #[test]
479 fn test_format_z_score_display() {
480 let test_cases = vec![
482 (2.5_f64, " 2.50"),
483 (0.0_f64, " 0.00"),
484 (-1.5_f64, " -1.50"),
485 (999.999_f64, " 1000.00"),
486 (0.001_f64, " 0.00"),
487 (f64::INFINITY, ""),
488 (f64::NEG_INFINITY, ""),
489 (f64::NAN, ""),
490 ];
491
492 for (z_score, expected) in test_cases {
493 let result = format_z_score_display(z_score);
494 assert_eq!(result, expected, "Failed for z_score: {}", z_score);
495 }
496 }
497
498 #[test]
499 fn test_direction_arrows() {
500 let test_cases = vec![
502 (5.0_f64, 3.0_f64, "↑"), (1.0_f64, 3.0_f64, "↓"), (3.0_f64, 3.0_f64, "→"), ];
506
507 for (head_mean, tail_mean, expected) in test_cases {
508 let result = get_direction_arrow(head_mean, tail_mean);
509 assert_eq!(
510 result, expected,
511 "Failed for head_mean: {}, tail_mean: {}",
512 head_mean, tail_mean
513 );
514 }
515 }
516
517 #[test]
518 fn test_audit_with_different_dispersion_methods() {
519 let head_value = 35.0;
523 let tail_values = [30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 100.0];
524
525 let head_summary = stats::aggregate_measurements(std::iter::once(&head_value));
526 let tail_summary = stats::aggregate_measurements(tail_values.iter());
527
528 let z_score_stddev =
530 head_summary.z_score_with_method(&tail_summary, DispersionMethod::StandardDeviation);
531 let z_score_mad = head_summary
532 .z_score_with_method(&tail_summary, DispersionMethod::MedianAbsoluteDeviation);
533
534 assert!(
537 z_score_stddev < z_score_mad,
538 "stddev z-score ({}) should be smaller than MAD z-score ({}) with outlier data",
539 z_score_stddev,
540 z_score_mad
541 );
542
543 assert!(z_score_stddev > 0.0);
545 assert!(z_score_mad > 0.0);
546 }
547
548 #[test]
549 fn test_dispersion_method_conversion() {
550 let cli_stddev = git_perf_cli_types::DispersionMethod::StandardDeviation;
554 let stats_stddev: DispersionMethod = cli_stddev.into();
555 assert_eq!(stats_stddev, DispersionMethod::StandardDeviation);
556
557 let cli_mad = git_perf_cli_types::DispersionMethod::MedianAbsoluteDeviation;
559 let stats_mad: DispersionMethod = cli_mad.into();
560 assert_eq!(stats_mad, DispersionMethod::MedianAbsoluteDeviation);
561 }
562
563 #[test]
564 fn test_audit_multiple_with_no_measurements() {
565 with_isolated_test_setup(|_git_dir, _home_path| {
569 let result = audit_multiple(
570 "HEAD",
571 100,
572 Some(1),
573 &[],
574 Some(ReductionFunc::Mean),
575 Some(2.0),
576 Some(DispersionMethod::StandardDeviation),
577 &[], false,
579 );
580
581 assert!(
583 result.is_ok(),
584 "audit_multiple should succeed with empty pattern list"
585 );
586 });
587 }
588
589 #[test]
592 fn test_min_count_boundary_condition() {
593 let result = audit_with_data(
596 "test_measurement",
597 15.0,
598 vec![10.0, 11.0, 12.0], 3, 2.0,
601 DispersionMethod::StandardDeviation,
602 );
603
604 assert!(result.is_ok());
605 let audit_result = result.unwrap();
606 assert!(!audit_result.message.contains("Skipping test"));
608
609 let result = audit_with_data(
611 "test_measurement",
612 15.0,
613 vec![10.0, 11.0], 3, 2.0,
616 DispersionMethod::StandardDeviation,
617 );
618
619 assert!(result.is_ok());
620 let audit_result = result.unwrap();
621 assert!(audit_result.message.contains("Skipping test"));
622 assert!(audit_result.passed); }
624
625 #[test]
626 fn test_pluralization_logic() {
627 let result = audit_with_data(
630 "test_measurement",
631 15.0,
632 vec![], 5, 2.0,
635 DispersionMethod::StandardDeviation,
636 );
637
638 assert!(result.is_ok());
639 let message = result.unwrap().message;
640 assert!(message.contains("0 historical measurements found")); assert!(!message.contains("0 historical measurement found")); let result = audit_with_data(
645 "test_measurement",
646 15.0,
647 vec![10.0], 5, 2.0,
650 DispersionMethod::StandardDeviation,
651 );
652
653 assert!(result.is_ok());
654 let message = result.unwrap().message;
655 assert!(message.contains("1 historical measurement found")); let result = audit_with_data(
659 "test_measurement",
660 15.0,
661 vec![10.0, 11.0], 5, 2.0,
664 DispersionMethod::StandardDeviation,
665 );
666
667 assert!(result.is_ok());
668 let message = result.unwrap().message;
669 assert!(message.contains("2 historical measurements found")); }
671
672 #[test]
673 fn test_skip_with_summaries() {
674 let result = audit_with_data(
680 "test_measurement",
681 15.0,
682 vec![], 5, 2.0,
685 DispersionMethod::StandardDeviation,
686 );
687
688 assert!(result.is_ok());
689 let message = result.unwrap().message;
690 assert!(message.contains("Skipping test"));
691 assert!(message.contains("Head:")); assert!(!message.contains("z-score")); assert!(!message.contains("Tail:")); assert!(!message.contains("[")); let result = audit_with_data(
698 "test_measurement",
699 15.0,
700 vec![10.0], 5, 2.0,
703 DispersionMethod::StandardDeviation,
704 );
705
706 assert!(result.is_ok());
707 let message = result.unwrap().message;
708 assert!(message.contains("Skipping test"));
709 assert!(message.contains("z-score (stddev):")); assert!(message.contains("Head:")); assert!(message.contains("Tail:")); assert!(message.contains("[")); let z_pos = message.find("z-score").unwrap();
715 let head_pos = message.find("Head:").unwrap();
716 let tail_pos = message.find("Tail:").unwrap();
717 let spark_pos = message.find("[").unwrap();
718 assert!(z_pos < head_pos, "z-score should come before Head");
719 assert!(head_pos < tail_pos, "Head should come before Tail");
720 assert!(tail_pos < spark_pos, "Tail should come before sparkline");
721
722 let result = audit_with_data(
724 "test_measurement",
725 15.0,
726 vec![10.0, 11.0], 5, 2.0,
729 DispersionMethod::StandardDeviation,
730 );
731
732 assert!(result.is_ok());
733 let message = result.unwrap().message;
734 assert!(message.contains("Skipping test"));
735 assert!(message.contains("z-score (stddev):")); assert!(message.contains("Head:")); assert!(message.contains("Tail:")); assert!(message.contains("[")); let z_pos = message.find("z-score").unwrap();
741 let head_pos = message.find("Head:").unwrap();
742 let tail_pos = message.find("Tail:").unwrap();
743 let spark_pos = message.find("[").unwrap();
744 assert!(z_pos < head_pos, "z-score should come before Head");
745 assert!(head_pos < tail_pos, "Head should come before Tail");
746 assert!(tail_pos < spark_pos, "Tail should come before sparkline");
747
748 let result = audit_with_data(
750 "test_measurement",
751 15.0,
752 vec![10.0, 11.0], 5, 2.0,
755 DispersionMethod::MedianAbsoluteDeviation,
756 );
757
758 assert!(result.is_ok());
759 let message = result.unwrap().message;
760 assert!(message.contains("z-score (mad):")); }
762
763 #[test]
764 fn test_relative_calculations_division_vs_modulo() {
765 let result = audit_with_data(
768 "test_measurement",
769 25.0, vec![10.0, 10.0, 10.0], 2,
772 10.0, DispersionMethod::StandardDeviation,
774 );
775
776 assert!(result.is_ok());
777 let audit_result = result.unwrap();
778
779 assert!(audit_result.message.contains("[+0.00% – +150.00%]"));
789
790 assert!(!audit_result.message.contains("[-100.00% – -50.00%]"));
792 assert!(!audit_result.message.contains("-100.00%"));
793 assert!(!audit_result.message.contains("-50.00%"));
794 }
795
796 #[test]
797 fn test_core_pass_fail_logic() {
798 let result = audit_with_data(
803 "test_measurement", 100.0, vec![10.0, 10.0, 10.0, 10.0, 10.0], 2,
807 0.5, DispersionMethod::StandardDeviation,
809 );
810
811 assert!(result.is_ok());
812 let audit_result = result.unwrap();
813 assert!(!audit_result.passed); assert!(audit_result.message.contains("❌"));
815
816 let result = audit_with_data(
818 "test_measurement",
819 10.2, vec![10.0, 10.1, 10.0, 10.1, 10.0], 2,
822 100.0, DispersionMethod::StandardDeviation,
824 );
825
826 assert!(result.is_ok());
827 let audit_result = result.unwrap();
828 assert!(audit_result.passed); assert!(audit_result.message.contains("✅"));
830 }
831
832 #[test]
833 fn test_final_result_logic() {
834 let result = audit_with_data(
839 "test_measurement",
840 1000.0, vec![10.0, 10.0, 10.0, 10.0, 10.0],
842 2,
843 0.1, DispersionMethod::StandardDeviation,
845 );
846
847 assert!(result.is_ok());
848 let audit_result = result.unwrap();
849 assert!(!audit_result.passed);
850 assert!(audit_result.message.contains("❌"));
851 assert!(audit_result.message.contains("differs significantly"));
852
853 let result = audit_with_data(
855 "test_measurement",
856 10.01, vec![10.0, 10.1, 10.0, 10.1, 10.0], 2,
859 100.0, DispersionMethod::StandardDeviation,
861 );
862
863 assert!(result.is_ok());
864 let audit_result = result.unwrap();
865 assert!(audit_result.passed);
866 assert!(audit_result.message.contains("✅"));
867 assert!(!audit_result.message.contains("differs significantly"));
868 }
869
870 #[test]
871 fn test_dispersion_methods_produce_different_results() {
872 let head = 35.0;
874 let tail = vec![30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 30.0, 100.0];
875
876 let result_stddev = audit_with_data(
877 "test_measurement",
878 head,
879 tail.clone(),
880 2,
881 2.0,
882 DispersionMethod::StandardDeviation,
883 );
884
885 let result_mad = audit_with_data(
886 "test_measurement",
887 head,
888 tail,
889 2,
890 2.0,
891 DispersionMethod::MedianAbsoluteDeviation,
892 );
893
894 assert!(result_stddev.is_ok());
895 assert!(result_mad.is_ok());
896
897 let stddev_result = result_stddev.unwrap();
898 let mad_result = result_mad.unwrap();
899
900 assert!(stddev_result.message.contains("stddev"));
902 assert!(mad_result.message.contains("mad"));
903 }
904
905 #[test]
906 fn test_head_and_tail_have_units_and_auto_scaling() {
907 use crate::test_helpers::setup_test_env_with_config;
911
912 let config_content = r#"
913[measurement."build_time"]
914unit = "ms"
915"#;
916 let (_temp_dir, _dir_guard) = setup_test_env_with_config(config_content);
917
918 let head = 12_345.67; let tail = vec![10_000.0, 10_500.0, 11_000.0, 11_500.0, 12_000.0]; let result = audit_with_data(
923 "build_time",
924 head,
925 tail,
926 2,
927 10.0, DispersionMethod::StandardDeviation,
929 );
930
931 assert!(result.is_ok());
932 let audit_result = result.unwrap();
933 let message = &audit_result.message;
934
935 assert!(
937 message.contains("Head:"),
938 "Message should contain Head section"
939 );
940
941 assert!(
944 message.contains("12.3s") || message.contains("12.35s"),
945 "Head mean should be auto-scaled to seconds, got: {}",
946 message
947 );
948
949 let head_section: Vec<&str> = message
950 .lines()
951 .filter(|line| line.contains("Head:"))
952 .collect();
953
954 assert!(
955 !head_section.is_empty(),
956 "Should find Head section in message"
957 );
958
959 let head_line = head_section[0];
960
961 assert!(
964 head_line.contains("μ:") && head_line.contains("σ:") && head_line.contains("MAD:"),
965 "Head line should contain μ, σ, and MAD labels, got: {}",
966 head_line
967 );
968
969 assert!(
971 message.contains("Tail:"),
972 "Message should contain Tail section"
973 );
974
975 let tail_section: Vec<&str> = message
976 .lines()
977 .filter(|line| line.contains("Tail:"))
978 .collect();
979
980 assert!(
981 !tail_section.is_empty(),
982 "Should find Tail section in message"
983 );
984
985 let tail_line = tail_section[0];
986
987 assert!(
989 tail_line.contains("11s")
990 || tail_line.contains("11.")
991 || tail_line.contains("10.")
992 || tail_line.contains("12."),
993 "Tail should contain auto-scaled second values, got: {}",
994 tail_line
995 );
996
997 assert!(
999 tail_line.contains("μ:")
1000 && tail_line.contains("σ:")
1001 && tail_line.contains("MAD:")
1002 && tail_line.contains("n:"),
1003 "Tail line should contain all stat labels, got: {}",
1004 tail_line
1005 );
1006 }
1007
1008 #[test]
1009 fn test_threshold_note_only_shown_when_audit_would_fail() {
1010 use crate::test_helpers::setup_test_env_with_config;
1013
1014 let config_content = r#"
1015[measurement."build_time"]
1016min_relative_deviation = 10.0
1017"#;
1018 let (_temp_dir, _dir_guard) = setup_test_env_with_config(config_content);
1019
1020 let result = audit_with_data(
1023 "build_time",
1024 10.1, vec![10.0, 10.1, 10.0, 10.1, 10.0], 2,
1027 100.0, DispersionMethod::StandardDeviation,
1029 );
1030
1031 assert!(result.is_ok());
1032 let audit_result = result.unwrap();
1033 assert!(audit_result.passed);
1034 assert!(audit_result.message.contains("✅"));
1035 assert!(
1037 !audit_result
1038 .message
1039 .contains("Note: Passed due to relative deviation"),
1040 "Note should not appear when audit passes without needing threshold bypass"
1041 );
1042
1043 let result = audit_with_data(
1046 "build_time",
1047 1002.0, vec![1000.0, 1000.1, 1000.0, 1000.1, 1000.0], 2,
1050 0.5, DispersionMethod::StandardDeviation,
1052 );
1053
1054 assert!(result.is_ok());
1055 let audit_result = result.unwrap();
1056 assert!(audit_result.passed);
1057 assert!(audit_result.message.contains("✅"));
1058 assert!(
1060 audit_result
1061 .message
1062 .contains("Note: Passed due to relative deviation"),
1063 "Note should appear when audit passes due to threshold bypass. Got: {}",
1064 audit_result.message
1065 );
1066
1067 let result = audit_with_data(
1070 "build_time",
1071 1200.0, vec![1000.0, 1000.1, 1000.0, 1000.1, 1000.0], 2,
1074 0.5, DispersionMethod::StandardDeviation,
1076 );
1077
1078 assert!(result.is_ok());
1079 let audit_result = result.unwrap();
1080 assert!(!audit_result.passed);
1081 assert!(audit_result.message.contains("❌"));
1082 assert!(
1084 !audit_result
1085 .message
1086 .contains("Note: Passed due to relative deviation"),
1087 "Note should not appear when audit fails"
1088 );
1089 }
1090
1091 #[cfg(test)]
1093 mod integration {
1094 use super::*;
1095 use crate::config::{
1096 audit_aggregate_by, audit_dispersion_method, audit_min_measurements, audit_sigma,
1097 };
1098 use crate::test_helpers::setup_test_env_with_config;
1099
1100 #[test]
1101 fn test_different_dispersion_methods_per_measurement() {
1102 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1103 r#"
1104[measurement]
1105dispersion_method = "stddev"
1106
1107[measurement."build_time"]
1108dispersion_method = "mad"
1109
1110[measurement."memory_usage"]
1111dispersion_method = "stddev"
1112"#,
1113 );
1114
1115 let build_time_method = audit_dispersion_method("build_time");
1117 let memory_usage_method = audit_dispersion_method("memory_usage");
1118 let other_method = audit_dispersion_method("other_metric");
1119
1120 assert_eq!(
1121 DispersionMethod::from(build_time_method),
1122 DispersionMethod::MedianAbsoluteDeviation,
1123 "build_time should use MAD"
1124 );
1125 assert_eq!(
1126 DispersionMethod::from(memory_usage_method),
1127 DispersionMethod::StandardDeviation,
1128 "memory_usage should use stddev"
1129 );
1130 assert_eq!(
1131 DispersionMethod::from(other_method),
1132 DispersionMethod::StandardDeviation,
1133 "other_metric should use default stddev"
1134 );
1135 }
1136
1137 #[test]
1138 fn test_different_min_measurements_per_measurement() {
1139 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1140 r#"
1141[measurement]
1142min_measurements = 5
1143
1144[measurement."build_time"]
1145min_measurements = 10
1146
1147[measurement."memory_usage"]
1148min_measurements = 3
1149"#,
1150 );
1151
1152 assert_eq!(
1153 audit_min_measurements("build_time"),
1154 Some(10),
1155 "build_time should require 10 measurements"
1156 );
1157 assert_eq!(
1158 audit_min_measurements("memory_usage"),
1159 Some(3),
1160 "memory_usage should require 3 measurements"
1161 );
1162 assert_eq!(
1163 audit_min_measurements("other_metric"),
1164 Some(5),
1165 "other_metric should use default 5 measurements"
1166 );
1167 }
1168
1169 #[test]
1170 fn test_different_aggregate_by_per_measurement() {
1171 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1172 r#"
1173[measurement]
1174aggregate_by = "median"
1175
1176[measurement."build_time"]
1177aggregate_by = "max"
1178
1179[measurement."memory_usage"]
1180aggregate_by = "mean"
1181"#,
1182 );
1183
1184 assert_eq!(
1185 audit_aggregate_by("build_time"),
1186 Some(git_perf_cli_types::ReductionFunc::Max),
1187 "build_time should use max"
1188 );
1189 assert_eq!(
1190 audit_aggregate_by("memory_usage"),
1191 Some(git_perf_cli_types::ReductionFunc::Mean),
1192 "memory_usage should use mean"
1193 );
1194 assert_eq!(
1195 audit_aggregate_by("other_metric"),
1196 Some(git_perf_cli_types::ReductionFunc::Median),
1197 "other_metric should use default median"
1198 );
1199 }
1200
1201 #[test]
1202 fn test_different_sigma_per_measurement() {
1203 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1204 r#"
1205[measurement]
1206sigma = 3.0
1207
1208[measurement."build_time"]
1209sigma = 5.5
1210
1211[measurement."memory_usage"]
1212sigma = 2.0
1213"#,
1214 );
1215
1216 assert_eq!(
1217 audit_sigma("build_time"),
1218 Some(5.5),
1219 "build_time should use sigma 5.5"
1220 );
1221 assert_eq!(
1222 audit_sigma("memory_usage"),
1223 Some(2.0),
1224 "memory_usage should use sigma 2.0"
1225 );
1226 assert_eq!(
1227 audit_sigma("other_metric"),
1228 Some(3.0),
1229 "other_metric should use default sigma 3.0"
1230 );
1231 }
1232
1233 #[test]
1234 fn test_cli_overrides_config() {
1235 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1236 r#"
1237[measurement."build_time"]
1238min_measurements = 10
1239aggregate_by = "max"
1240sigma = 5.5
1241dispersion_method = "mad"
1242"#,
1243 );
1244
1245 let params = super::resolve_audit_params(
1247 "build_time",
1248 Some(2), Some(ReductionFunc::Min), Some(3.0), Some(DispersionMethod::StandardDeviation), );
1253
1254 assert_eq!(
1255 params.min_count, 2,
1256 "CLI min_measurements should override config"
1257 );
1258 assert_eq!(
1259 params.summarize_by,
1260 ReductionFunc::Min,
1261 "CLI aggregate_by should override config"
1262 );
1263 assert_eq!(params.sigma, 3.0, "CLI sigma should override config");
1264 assert_eq!(
1265 params.dispersion_method,
1266 DispersionMethod::StandardDeviation,
1267 "CLI dispersion should override config"
1268 );
1269 }
1270
1271 #[test]
1272 fn test_config_overrides_defaults() {
1273 let (_temp_dir, _dir_guard) = setup_test_env_with_config(
1274 r#"
1275[measurement."build_time"]
1276min_measurements = 10
1277aggregate_by = "max"
1278sigma = 5.5
1279dispersion_method = "mad"
1280"#,
1281 );
1282
1283 let params = super::resolve_audit_params(
1285 "build_time",
1286 None, None,
1288 None,
1289 None,
1290 );
1291
1292 assert_eq!(
1293 params.min_count, 10,
1294 "Config min_measurements should override default"
1295 );
1296 assert_eq!(
1297 params.summarize_by,
1298 ReductionFunc::Max,
1299 "Config aggregate_by should override default"
1300 );
1301 assert_eq!(params.sigma, 5.5, "Config sigma should override default");
1302 assert_eq!(
1303 params.dispersion_method,
1304 DispersionMethod::MedianAbsoluteDeviation,
1305 "Config dispersion should override default"
1306 );
1307 }
1308
1309 #[test]
1310 fn test_uses_defaults_when_no_config_or_cli() {
1311 let (_temp_dir, _dir_guard) = setup_test_env_with_config("");
1312
1313 let params = super::resolve_audit_params(
1315 "non_existent_measurement",
1316 None, None,
1318 None,
1319 None,
1320 );
1321
1322 assert_eq!(
1323 params.min_count, 2,
1324 "Should use default min_measurements of 2"
1325 );
1326 assert_eq!(
1327 params.summarize_by,
1328 ReductionFunc::Min,
1329 "Should use default aggregate_by of Min"
1330 );
1331 assert_eq!(params.sigma, 4.0, "Should use default sigma of 4.0");
1332 assert_eq!(
1333 params.dispersion_method,
1334 DispersionMethod::StandardDeviation,
1335 "Should use default dispersion of stddev"
1336 );
1337 }
1338 }
1339
1340 #[test]
1341 fn test_discover_matching_measurements() {
1342 use crate::data::{Commit, MeasurementData};
1343 use std::collections::HashMap;
1344
1345 let commits = vec![
1347 Ok(Commit {
1348 commit: "abc123".to_string(),
1349 title: "test: commit 1".to_string(),
1350 author: "Test Author".to_string(),
1351 measurements: vec![
1352 MeasurementData {
1353 epoch: 0,
1354 name: "bench_cpu".to_string(),
1355 timestamp: 1000.0,
1356 val: 100.0,
1357 key_values: {
1358 let mut map = HashMap::new();
1359 map.insert("os".to_string(), "linux".to_string());
1360 map
1361 },
1362 },
1363 MeasurementData {
1364 epoch: 0,
1365 name: "bench_memory".to_string(),
1366 timestamp: 1000.0,
1367 val: 200.0,
1368 key_values: {
1369 let mut map = HashMap::new();
1370 map.insert("os".to_string(), "linux".to_string());
1371 map
1372 },
1373 },
1374 MeasurementData {
1375 epoch: 0,
1376 name: "test_unit".to_string(),
1377 timestamp: 1000.0,
1378 val: 50.0,
1379 key_values: {
1380 let mut map = HashMap::new();
1381 map.insert("os".to_string(), "linux".to_string());
1382 map
1383 },
1384 },
1385 ],
1386 }),
1387 Ok(Commit {
1388 commit: "def456".to_string(),
1389 title: "test: commit 2".to_string(),
1390 author: "Test Author".to_string(),
1391 measurements: vec![
1392 MeasurementData {
1393 epoch: 0,
1394 name: "bench_cpu".to_string(),
1395 timestamp: 1000.0,
1396 val: 105.0,
1397 key_values: {
1398 let mut map = HashMap::new();
1399 map.insert("os".to_string(), "mac".to_string());
1400 map
1401 },
1402 },
1403 MeasurementData {
1404 epoch: 0,
1405 name: "other_metric".to_string(),
1406 timestamp: 1000.0,
1407 val: 75.0,
1408 key_values: {
1409 let mut map = HashMap::new();
1410 map.insert("os".to_string(), "linux".to_string());
1411 map
1412 },
1413 },
1414 ],
1415 }),
1416 ];
1417
1418 let patterns = vec!["bench_.*".to_string()];
1420 let filters = crate::filter::compile_filters(&patterns).unwrap();
1421 let selectors = vec![];
1422 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1423
1424 assert_eq!(discovered.len(), 2);
1425 assert!(discovered.contains(&"bench_cpu".to_string()));
1426 assert!(discovered.contains(&"bench_memory".to_string()));
1427 assert!(!discovered.contains(&"test_unit".to_string()));
1428 assert!(!discovered.contains(&"other_metric".to_string()));
1429
1430 let patterns = vec!["bench_cpu".to_string(), "test_.*".to_string()];
1432 let filters = crate::filter::compile_filters(&patterns).unwrap();
1433 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1434
1435 assert_eq!(discovered.len(), 2);
1436 assert!(discovered.contains(&"bench_cpu".to_string()));
1437 assert!(discovered.contains(&"test_unit".to_string()));
1438 assert!(!discovered.contains(&"bench_memory".to_string()));
1439
1440 let patterns = vec!["bench_.*".to_string()];
1442 let filters = crate::filter::compile_filters(&patterns).unwrap();
1443 let selectors = vec![("os".to_string(), "linux".to_string())];
1444 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1445
1446 assert_eq!(discovered.len(), 2);
1449 assert!(discovered.contains(&"bench_cpu".to_string()));
1450 assert!(discovered.contains(&"bench_memory".to_string()));
1451
1452 let patterns = vec!["nonexistent.*".to_string()];
1454 let filters = crate::filter::compile_filters(&patterns).unwrap();
1455 let selectors = vec![];
1456 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1457
1458 assert_eq!(discovered.len(), 0);
1459
1460 let filters = vec![];
1462 let selectors = vec![];
1463 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1464
1465 assert_eq!(discovered.len(), 4);
1469 assert!(discovered.contains(&"bench_cpu".to_string()));
1470 assert!(discovered.contains(&"bench_memory".to_string()));
1471 assert!(discovered.contains(&"test_unit".to_string()));
1472 assert!(discovered.contains(&"other_metric".to_string()));
1473
1474 let patterns = vec!["bench_.*".to_string()];
1476 let filters = crate::filter::compile_filters(&patterns).unwrap();
1477 let selectors = vec![("os".to_string(), "windows".to_string())];
1478 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1479
1480 assert_eq!(discovered.len(), 0);
1481
1482 let patterns = vec!["^bench_cpu$".to_string()];
1484 let filters = crate::filter::compile_filters(&patterns).unwrap();
1485 let selectors = vec![];
1486 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1487
1488 assert_eq!(discovered.len(), 1);
1489 assert!(discovered.contains(&"bench_cpu".to_string()));
1490
1491 let patterns = vec![".*".to_string()]; let filters = crate::filter::compile_filters(&patterns).unwrap();
1494 let selectors = vec![];
1495 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1496
1497 assert_eq!(discovered[0], "bench_cpu");
1499 assert_eq!(discovered[1], "bench_memory");
1500 assert_eq!(discovered[2], "other_metric");
1501 assert_eq!(discovered[3], "test_unit");
1502 }
1503
1504 #[test]
1505 fn test_audit_multiple_with_combined_patterns() {
1506 use crate::data::{Commit, MeasurementData};
1513 use std::collections::HashMap;
1514
1515 let commits = vec![Ok(Commit {
1517 commit: "abc123".to_string(),
1518 title: "test: commit".to_string(),
1519 author: "Test Author".to_string(),
1520 measurements: vec![
1521 MeasurementData {
1522 epoch: 0,
1523 name: "timer".to_string(),
1524 timestamp: 1000.0,
1525 val: 10.0,
1526 key_values: HashMap::new(),
1527 },
1528 MeasurementData {
1529 epoch: 0,
1530 name: "bench_cpu".to_string(),
1531 timestamp: 1000.0,
1532 val: 100.0,
1533 key_values: HashMap::new(),
1534 },
1535 MeasurementData {
1536 epoch: 0,
1537 name: "memory".to_string(),
1538 timestamp: 1000.0,
1539 val: 500.0,
1540 key_values: HashMap::new(),
1541 },
1542 ],
1543 })];
1544
1545 let measurements = vec!["timer".to_string()];
1548 let filter_patterns = vec!["bench_.*".to_string()];
1549 let combined =
1550 crate::filter::combine_measurements_and_filters(&measurements, &filter_patterns);
1551
1552 assert_eq!(combined.len(), 2);
1554 assert_eq!(combined[0], "^timer$");
1555 assert_eq!(combined[1], "bench_.*");
1556
1557 let filters = crate::filter::compile_filters(&combined).unwrap();
1559 let selectors = vec![];
1560 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1561
1562 assert_eq!(discovered.len(), 2);
1564 assert!(discovered.contains(&"timer".to_string()));
1565 assert!(discovered.contains(&"bench_cpu".to_string()));
1566 assert!(!discovered.contains(&"memory".to_string())); let measurements = vec!["timer".to_string(), "memory".to_string()];
1570 let filter_patterns = vec!["bench_.*".to_string(), "test_.*".to_string()];
1571 let combined =
1572 crate::filter::combine_measurements_and_filters(&measurements, &filter_patterns);
1573
1574 assert_eq!(combined.len(), 4);
1575
1576 let filters = crate::filter::compile_filters(&combined).unwrap();
1577 let discovered = discover_matching_measurements(&commits, &filters, &selectors);
1578
1579 assert_eq!(discovered.len(), 3);
1581 assert!(discovered.contains(&"timer".to_string()));
1582 assert!(discovered.contains(&"memory".to_string()));
1583 assert!(discovered.contains(&"bench_cpu".to_string()));
1584 }
1585
1586 #[test]
1587 fn test_audit_with_empty_tail() {
1588 let result = audit_with_data(
1592 "test_measurement",
1593 10.0, vec![], 2, 2.0, DispersionMethod::StandardDeviation,
1598 );
1599
1600 assert!(result.is_ok(), "Should not crash on empty tail");
1602 let audit_result = result.unwrap();
1603
1604 assert!(audit_result.passed);
1606 assert!(audit_result.message.contains("Skipping test"));
1607
1608 assert!(!audit_result.message.to_lowercase().contains("inf"));
1610 assert!(!audit_result.message.to_lowercase().contains("nan"));
1611 }
1612
1613 #[test]
1614 fn test_audit_with_all_zero_tail() {
1615 let result = audit_with_data(
1618 "test_measurement",
1619 5.0, vec![0.0, 0.0, 0.0], 2, 2.0, DispersionMethod::StandardDeviation,
1624 );
1625
1626 assert!(result.is_ok(), "Should not crash when tail median is 0.0");
1628 let audit_result = result.unwrap();
1629
1630 assert!(!audit_result.message.to_lowercase().contains("inf"));
1632 assert!(!audit_result.message.to_lowercase().contains("nan"));
1633 }
1634
1635 #[test]
1636 fn test_tiered_baseline_approach() {
1637 let result = audit_with_data(
1643 "test_measurement",
1644 15.0, vec![10.0, 11.0, 12.0], 2,
1647 2.0,
1648 DispersionMethod::StandardDeviation,
1649 );
1650
1651 assert!(result.is_ok());
1652 let audit_result = result.unwrap();
1653 assert!(audit_result.message.contains('%'));
1655 assert!(!audit_result.message.to_lowercase().contains("inf"));
1656
1657 let result = audit_with_data(
1659 "test_measurement",
1660 5.0, vec![0.0, 0.0, 0.0], 2,
1663 2.0,
1664 DispersionMethod::StandardDeviation,
1665 );
1666
1667 assert!(result.is_ok());
1668 let audit_result = result.unwrap();
1669 assert!(!audit_result.message.to_lowercase().contains("inf"));
1672 assert!(!audit_result.message.to_lowercase().contains("nan"));
1673 assert!(audit_result.message.contains('–') || audit_result.message.contains('-'));
1675
1676 let result = audit_with_data(
1678 "test_measurement",
1679 0.0, vec![0.0, 0.0, 0.0], 2,
1682 2.0,
1683 DispersionMethod::StandardDeviation,
1684 );
1685
1686 assert!(result.is_ok());
1687 let audit_result = result.unwrap();
1688 assert!(!audit_result.message.to_lowercase().contains("inf"));
1690 assert!(!audit_result.message.to_lowercase().contains("nan"));
1691 }
1692
1693 #[test]
1694 fn test_min_measurements_two_with_no_tail() {
1695 let result = audit_with_data(
1698 "test_measurement",
1699 15.0, vec![], 2, 2.0,
1703 DispersionMethod::StandardDeviation,
1704 );
1705
1706 assert!(result.is_ok());
1707 let audit_result = result.unwrap();
1708
1709 assert!(audit_result.passed);
1711 assert!(audit_result.message.contains("Skipping test"));
1712 assert!(audit_result
1713 .message
1714 .contains("0 historical measurements found"));
1715 assert!(audit_result
1716 .message
1717 .contains("Less than requested min_measurements of 2"));
1718
1719 assert!(audit_result.message.contains("Head:"));
1721 assert!(!audit_result.message.contains("z-score"));
1722 assert!(!audit_result.message.contains("Tail:"));
1723 }
1724
1725 #[test]
1726 fn test_min_measurements_two_with_single_tail() {
1727 let result = audit_with_data(
1730 "test_measurement",
1731 15.0, vec![10.0], 2, 2.0,
1735 DispersionMethod::StandardDeviation,
1736 );
1737
1738 assert!(result.is_ok());
1739 let audit_result = result.unwrap();
1740
1741 assert!(audit_result.passed);
1743 assert!(audit_result.message.contains("Skipping test"));
1744 assert!(audit_result
1745 .message
1746 .contains("1 historical measurement found"));
1747 assert!(audit_result
1748 .message
1749 .contains("Less than requested min_measurements of 2"));
1750
1751 assert!(audit_result.message.contains("Head:"));
1753 assert!(audit_result.message.contains("Tail:"));
1754 assert!(audit_result.message.contains("z-score"));
1755 assert!(audit_result.message.contains("["));
1756 }
1757}