1use std::collections::{HashMap, HashSet};
10use std::fmt;
11
12use thiserror::Error;
13
14mod execution;
15
16pub use execution::{
17 AdapterRun, CollectedOutput, ExecutionStage, ProviderAdapter, ProviderEngine,
18 ProviderExecutionError, ProviderRun, StartedRun,
19};
20
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum MatrixOutcome {
24 Complete,
26 Partial,
28 Failed,
30}
31
32#[derive(Clone, Debug, PartialEq, Eq)]
34pub struct ExpectedSession {
35 pub session_id: String,
37 pub device_id: String,
39 pub status: String,
41}
42
43#[derive(Clone, Debug, PartialEq, Eq)]
45pub struct CollectedSession {
46 pub session_id: String,
48 pub report_count: usize,
50 pub failure: Option<String>,
52}
53
54#[derive(Clone, Debug, PartialEq, Eq)]
56pub enum SessionDisposition {
57 Complete,
59 Missing,
61 NonPassed {
63 status: String,
65 failure: Option<String>,
67 },
68 Resultless {
70 failure: Option<String>,
72 },
73}
74
75#[derive(Clone, Debug, PartialEq, Eq)]
77pub struct SessionAssessment {
78 pub session_id: String,
80 pub device_id: String,
82 pub disposition: SessionDisposition,
84}
85
86#[derive(Clone, Debug, PartialEq, Eq)]
88pub struct MatrixAssessment {
89 outcome: MatrixOutcome,
90 sessions: Vec<SessionAssessment>,
91}
92
93impl MatrixAssessment {
94 pub const fn outcome(&self) -> MatrixOutcome {
96 self.outcome
97 }
98
99 pub fn sessions(&self) -> &[SessionAssessment] {
101 &self.sessions
102 }
103
104 pub const fn is_complete(&self) -> bool {
106 matches!(self.outcome, MatrixOutcome::Complete)
107 }
108}
109
110impl fmt::Display for MatrixAssessment {
111 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
112 let missing = self
113 .sessions
114 .iter()
115 .filter(|session| matches!(session.disposition, SessionDisposition::Missing))
116 .map(session_label)
117 .collect::<Vec<_>>();
118 let non_passed = self
119 .sessions
120 .iter()
121 .filter_map(|session| match &session.disposition {
122 SessionDisposition::NonPassed { status, failure } => Some(format!(
123 "{} ({}, status={}{}{})",
124 session.device_id,
125 session.session_id,
126 status,
127 failure.as_ref().map(|_| "; ").unwrap_or_default(),
128 failure.as_deref().unwrap_or_default()
129 )),
130 _ => None,
131 })
132 .collect::<Vec<_>>();
133 let resultless = self
134 .sessions
135 .iter()
136 .filter_map(|session| match &session.disposition {
137 SessionDisposition::Resultless { failure } => Some(format!(
138 "{} ({}{}{})",
139 session.device_id,
140 session.session_id,
141 failure.as_ref().map(|_| "; ").unwrap_or_default(),
142 failure.as_deref().unwrap_or_default()
143 )),
144 _ => None,
145 })
146 .collect::<Vec<_>>();
147
148 let mut groups = Vec::new();
149 if !missing.is_empty() {
150 groups.push(format!(
151 "missing collected sessions: {}",
152 missing.join(", ")
153 ));
154 }
155 if !non_passed.is_empty() {
156 groups.push(format!("non-passed sessions: {}", non_passed.join(", ")));
157 }
158 if !resultless.is_empty() {
159 groups.push(format!("result-less sessions: {}", resultless.join(", ")));
160 }
161 write!(formatter, "{}", groups.join("; "))
162 }
163}
164
165fn session_label(session: &SessionAssessment) -> String {
166 format!("{} ({})", session.device_id, session.session_id)
167}
168
169#[derive(Clone, Debug, Error, PartialEq, Eq)]
171pub enum ReconcileError {
172 #[error("terminal provider build reported no device sessions")]
174 NoExpectedSessions,
175 #[error("duplicate expected session: {session_id}")]
177 DuplicateExpectedSession { session_id: String },
178 #[error("duplicate expected device: {device_id}")]
180 DuplicateExpectedDevice { device_id: String },
181 #[error("duplicate collected session: {session_id}")]
183 DuplicateCollectedSession { session_id: String },
184 #[error("unexpected collected session: {session_id}")]
186 UnexpectedCollectedSession { session_id: String },
187 #[error("duplicate benchmark results: {device_id} ({session_id}, got {report_count})")]
189 DuplicateReports {
190 device_id: String,
191 session_id: String,
192 report_count: usize,
193 },
194}
195
196pub fn reconcile_sessions(
203 expected: &[ExpectedSession],
204 collected: &[CollectedSession],
205) -> Result<MatrixAssessment, ReconcileError> {
206 if expected.is_empty() {
207 return Err(ReconcileError::NoExpectedSessions);
208 }
209
210 let mut expected_session_ids = HashSet::with_capacity(expected.len());
211 let mut expected_device_ids = HashSet::with_capacity(expected.len());
212 for session in expected {
213 if !expected_session_ids.insert(session.session_id.as_str()) {
214 return Err(ReconcileError::DuplicateExpectedSession {
215 session_id: session.session_id.clone(),
216 });
217 }
218 if !expected_device_ids.insert(session.device_id.as_str()) {
219 return Err(ReconcileError::DuplicateExpectedDevice {
220 device_id: session.device_id.clone(),
221 });
222 }
223 }
224
225 let mut collected_by_session = HashMap::with_capacity(collected.len());
226 for session in collected {
227 if !expected_session_ids.contains(session.session_id.as_str()) {
228 return Err(ReconcileError::UnexpectedCollectedSession {
229 session_id: session.session_id.clone(),
230 });
231 }
232 if collected_by_session
233 .insert(session.session_id.as_str(), session)
234 .is_some()
235 {
236 return Err(ReconcileError::DuplicateCollectedSession {
237 session_id: session.session_id.clone(),
238 });
239 }
240 }
241
242 let mut completed = 0usize;
243 let mut sessions = Vec::with_capacity(expected.len());
244 for session in expected {
245 let disposition = match collected_by_session.get(session.session_id.as_str()) {
246 None => SessionDisposition::Missing,
247 Some(collected) if collected.report_count > 1 => {
248 return Err(ReconcileError::DuplicateReports {
249 device_id: session.device_id.clone(),
250 session_id: session.session_id.clone(),
251 report_count: collected.report_count,
252 });
253 }
254 Some(collected) if !session.status.eq_ignore_ascii_case("passed") => {
255 SessionDisposition::NonPassed {
256 status: session.status.clone(),
257 failure: collected.failure.clone(),
258 }
259 }
260 Some(collected) if collected.report_count == 0 => SessionDisposition::Resultless {
261 failure: collected.failure.clone(),
262 },
263 Some(_) => {
264 completed += 1;
265 SessionDisposition::Complete
266 }
267 };
268 sessions.push(SessionAssessment {
269 session_id: session.session_id.clone(),
270 device_id: session.device_id.clone(),
271 disposition,
272 });
273 }
274
275 let outcome = if completed == expected.len() {
276 MatrixOutcome::Complete
277 } else if completed == 0 {
278 MatrixOutcome::Failed
279 } else {
280 MatrixOutcome::Partial
281 };
282 Ok(MatrixAssessment { outcome, sessions })
283}
284
285#[cfg(test)]
286mod tests {
287 use super::*;
288
289 fn expected(count: usize) -> Vec<ExpectedSession> {
290 (0..count)
291 .map(|index| ExpectedSession {
292 session_id: format!("session-{index}"),
293 device_id: format!("device-{index}"),
294 status: "passed".to_owned(),
295 })
296 .collect()
297 }
298
299 fn collected(count: usize) -> Vec<CollectedSession> {
300 (0..count)
301 .rev()
302 .map(|index| CollectedSession {
303 session_id: format!("session-{index}"),
304 report_count: 1,
305 failure: None,
306 })
307 .collect()
308 }
309
310 #[test]
311 fn complete_matrices_are_deterministic_at_one_ten_and_fifty_devices() {
312 for count in [1, 10, 50] {
313 let expected = expected(count);
314 let assessment = reconcile_sessions(&expected, &collected(count)).unwrap();
315 assert_eq!(assessment.outcome(), MatrixOutcome::Complete);
316 assert!(assessment.is_complete());
317 assert_eq!(assessment.sessions().len(), count);
318 for (index, session) in assessment.sessions().iter().enumerate() {
319 assert_eq!(session.session_id, format!("session-{index}"));
320 assert_eq!(session.disposition, SessionDisposition::Complete);
321 }
322 }
323 }
324
325 #[test]
326 fn one_of_ten_is_partial_and_never_complete() {
327 let assessment = reconcile_sessions(&expected(10), &collected(1)).unwrap();
328 assert_eq!(assessment.outcome(), MatrixOutcome::Partial);
329 assert!(!assessment.is_complete());
330 assert!(
331 assessment
332 .to_string()
333 .contains("missing collected sessions")
334 );
335 assert!(assessment.to_string().contains("session-9"));
336 }
337
338 #[test]
339 fn zero_successful_sessions_is_failed() {
340 let mut expected = expected(1);
341 expected[0].status = "failed".to_owned();
342 let collected = [CollectedSession {
343 session_id: "session-0".to_owned(),
344 report_count: 0,
345 failure: Some("benchmark timed out".to_owned()),
346 }];
347 let assessment = reconcile_sessions(&expected, &collected).unwrap();
348 assert_eq!(assessment.outcome(), MatrixOutcome::Failed);
349 assert!(assessment.to_string().contains("non-passed sessions"));
350 assert!(assessment.to_string().contains("benchmark timed out"));
351 }
352
353 #[test]
354 fn passed_session_without_a_report_is_failed() {
355 let collected = [CollectedSession {
356 session_id: "session-0".to_owned(),
357 report_count: 0,
358 failure: Some("no report marker".to_owned()),
359 }];
360 let assessment = reconcile_sessions(&expected(1), &collected).unwrap();
361 assert_eq!(assessment.outcome(), MatrixOutcome::Failed);
362 assert!(assessment.to_string().contains("result-less sessions"));
363 }
364
365 #[test]
366 fn ambiguity_and_unexpected_sessions_fail_closed() {
367 let mut duplicate_expected = expected(2);
368 duplicate_expected[1].device_id = duplicate_expected[0].device_id.clone();
369 assert!(matches!(
370 reconcile_sessions(&duplicate_expected, &collected(2)),
371 Err(ReconcileError::DuplicateExpectedDevice { .. })
372 ));
373
374 let duplicate_collected = [collected(1)[0].clone(), collected(1)[0].clone()];
375 assert!(matches!(
376 reconcile_sessions(&expected(1), &duplicate_collected),
377 Err(ReconcileError::DuplicateCollectedSession { .. })
378 ));
379
380 let unexpected = [CollectedSession {
381 session_id: "not-requested".to_owned(),
382 report_count: 1,
383 failure: None,
384 }];
385 assert!(matches!(
386 reconcile_sessions(&expected(1), &unexpected),
387 Err(ReconcileError::UnexpectedCollectedSession { .. })
388 ));
389 }
390
391 #[test]
392 fn multiple_reports_for_one_session_are_ambiguous() {
393 let collected = [CollectedSession {
394 session_id: "session-0".to_owned(),
395 report_count: 2,
396 failure: None,
397 }];
398 assert_eq!(
399 reconcile_sessions(&expected(1), &collected),
400 Err(ReconcileError::DuplicateReports {
401 device_id: "device-0".to_owned(),
402 session_id: "session-0".to_owned(),
403 report_count: 2,
404 })
405 );
406 }
407}