1use crate::{EvalError, TimingStats, summarize_durations};
2
3#[derive(Debug, Clone, Copy, PartialEq)]
4pub struct PipelineDurations<'a> {
5 pub detect: &'a [std::time::Duration],
6 pub track: &'a [std::time::Duration],
7 pub recognize: &'a [std::time::Duration],
8 pub end_to_end: &'a [std::time::Duration],
9}
10
11#[derive(Debug, Clone, Copy, PartialEq)]
12pub struct PipelineBenchmarkReport {
13 pub frames: usize,
14 pub detect: TimingStats,
15 pub track: TimingStats,
16 pub recognize: TimingStats,
17 pub end_to_end: TimingStats,
18}
19
20#[derive(Debug, Clone, Copy, PartialEq, Default)]
21pub struct StageThresholds {
22 pub min_fps: Option<f64>,
23 pub max_mean_ms: Option<f64>,
24 pub max_p95_ms: Option<f64>,
25}
26
27#[derive(Debug, Clone, Copy, PartialEq, Default)]
28pub struct PipelineBenchmarkThresholds {
29 pub detect: StageThresholds,
30 pub track: StageThresholds,
31 pub recognize: StageThresholds,
32 pub end_to_end: StageThresholds,
33}
34
35#[derive(Debug, Clone, PartialEq)]
36pub struct BenchmarkViolation {
37 pub stage: &'static str,
38 pub metric: &'static str,
39 pub expected: f64,
40 pub observed: f64,
41}
42
43pub fn summarize_pipeline_durations(
44 durations: PipelineDurations<'_>,
45) -> Result<PipelineBenchmarkReport, EvalError> {
46 let frames = durations.end_to_end.len();
47 if frames == 0 {
48 return Err(EvalError::EmptyDurationSeries);
49 }
50
51 validate_series_len(frames, durations.detect.len(), "detect")?;
52 validate_series_len(frames, durations.track.len(), "track")?;
53 validate_series_len(frames, durations.recognize.len(), "recognize")?;
54
55 Ok(PipelineBenchmarkReport {
56 frames,
57 detect: summarize_durations(durations.detect)?,
58 track: summarize_durations(durations.track)?,
59 recognize: summarize_durations(durations.recognize)?,
60 end_to_end: summarize_durations(durations.end_to_end)?,
61 })
62}
63
64pub fn parse_pipeline_benchmark_thresholds(
65 text: &str,
66) -> Result<PipelineBenchmarkThresholds, EvalError> {
67 let mut thresholds = PipelineBenchmarkThresholds::default();
68
69 for (line_idx, raw_line) in text.lines().enumerate() {
70 let line_no = line_idx + 1;
71 let line = raw_line.trim();
72 if line.is_empty() || line.starts_with('#') {
73 continue;
74 }
75 let (key, value_str) =
76 line.split_once('=')
77 .ok_or_else(|| EvalError::InvalidThresholdEntry {
78 line: line_no,
79 message: "expected `stage.metric=value` entry".to_string(),
80 })?;
81 let (stage, metric) =
82 key.split_once('.')
83 .ok_or_else(|| EvalError::InvalidThresholdEntry {
84 line: line_no,
85 message: "expected key in form `stage.metric`".to_string(),
86 })?;
87 let value =
88 value_str
89 .trim()
90 .parse::<f64>()
91 .map_err(|_| EvalError::InvalidThresholdEntry {
92 line: line_no,
93 message: format!("invalid numeric value `{}`", value_str.trim()),
94 })?;
95 if !value.is_finite() || value < 0.0 {
96 return Err(EvalError::InvalidThresholdEntry {
97 line: line_no,
98 message: format!(
99 "threshold value must be finite and non-negative, got {}",
100 value_str.trim()
101 ),
102 });
103 }
104
105 let stage_thresholds = match stage {
106 "detect" => &mut thresholds.detect,
107 "track" => &mut thresholds.track,
108 "recognize" => &mut thresholds.recognize,
109 "end_to_end" => &mut thresholds.end_to_end,
110 _ => {
111 return Err(EvalError::InvalidThresholdEntry {
112 line: line_no,
113 message: format!("unsupported stage `{stage}`"),
114 });
115 }
116 };
117
118 match metric {
119 "min_fps" => stage_thresholds.min_fps = Some(value),
120 "max_mean_ms" => stage_thresholds.max_mean_ms = Some(value),
121 "max_p95_ms" => stage_thresholds.max_p95_ms = Some(value),
122 _ => {
123 return Err(EvalError::InvalidThresholdEntry {
124 line: line_no,
125 message: format!("unsupported metric `{metric}`"),
126 });
127 }
128 }
129 }
130
131 Ok(thresholds)
132}
133
134pub fn validate_pipeline_benchmark_thresholds(
135 report: &PipelineBenchmarkReport,
136 thresholds: &PipelineBenchmarkThresholds,
137) -> Vec<BenchmarkViolation> {
138 let mut violations = Vec::new();
139 validate_stage_thresholds(
140 "detect",
141 &report.detect,
142 &thresholds.detect,
143 &mut violations,
144 );
145 validate_stage_thresholds("track", &report.track, &thresholds.track, &mut violations);
146 validate_stage_thresholds(
147 "recognize",
148 &report.recognize,
149 &thresholds.recognize,
150 &mut violations,
151 );
152 validate_stage_thresholds(
153 "end_to_end",
154 &report.end_to_end,
155 &thresholds.end_to_end,
156 &mut violations,
157 );
158 violations
159}
160
161fn validate_series_len(expected: usize, got: usize, series: &'static str) -> Result<(), EvalError> {
162 if expected != got {
163 return Err(EvalError::DurationSeriesLengthMismatch {
164 expected,
165 got,
166 series,
167 });
168 }
169 Ok(())
170}
171
172fn validate_stage_thresholds(
173 stage: &'static str,
174 stats: &TimingStats,
175 thresholds: &StageThresholds,
176 violations: &mut Vec<BenchmarkViolation>,
177) {
178 if let Some(min_fps) = thresholds.min_fps
179 && stats.throughput_fps < min_fps
180 {
181 violations.push(BenchmarkViolation {
182 stage,
183 metric: "min_fps",
184 expected: min_fps,
185 observed: stats.throughput_fps,
186 });
187 }
188 if let Some(max_mean_ms) = thresholds.max_mean_ms
189 && stats.mean_ms > max_mean_ms
190 {
191 violations.push(BenchmarkViolation {
192 stage,
193 metric: "max_mean_ms",
194 expected: max_mean_ms,
195 observed: stats.mean_ms,
196 });
197 }
198 if let Some(max_p95_ms) = thresholds.max_p95_ms
199 && stats.p95_ms > max_p95_ms
200 {
201 violations.push(BenchmarkViolation {
202 stage,
203 metric: "max_p95_ms",
204 expected: max_p95_ms,
205 observed: stats.p95_ms,
206 });
207 }
208}