use std::collections::{HashMap, HashSet};
use std::fmt;
use thiserror::Error;
mod execution;
pub use execution::{
AdapterRun, CollectedOutput, ExecutionStage, ProviderAdapter, ProviderEngine,
ProviderExecutionError, ProviderRun, StartedRun,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MatrixOutcome {
Complete,
Partial,
Failed,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ExpectedSession {
pub session_id: String,
pub device_id: String,
pub status: String,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CollectedSession {
pub session_id: String,
pub report_count: usize,
pub failure: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum SessionDisposition {
Complete,
Missing,
NonPassed {
status: String,
failure: Option<String>,
},
Resultless {
failure: Option<String>,
},
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SessionAssessment {
pub session_id: String,
pub device_id: String,
pub disposition: SessionDisposition,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MatrixAssessment {
outcome: MatrixOutcome,
sessions: Vec<SessionAssessment>,
}
impl MatrixAssessment {
pub const fn outcome(&self) -> MatrixOutcome {
self.outcome
}
pub fn sessions(&self) -> &[SessionAssessment] {
&self.sessions
}
pub const fn is_complete(&self) -> bool {
matches!(self.outcome, MatrixOutcome::Complete)
}
}
impl fmt::Display for MatrixAssessment {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let missing = self
.sessions
.iter()
.filter(|session| matches!(session.disposition, SessionDisposition::Missing))
.map(session_label)
.collect::<Vec<_>>();
let non_passed = self
.sessions
.iter()
.filter_map(|session| match &session.disposition {
SessionDisposition::NonPassed { status, failure } => Some(format!(
"{} ({}, status={}{}{})",
session.device_id,
session.session_id,
status,
failure.as_ref().map(|_| "; ").unwrap_or_default(),
failure.as_deref().unwrap_or_default()
)),
_ => None,
})
.collect::<Vec<_>>();
let resultless = self
.sessions
.iter()
.filter_map(|session| match &session.disposition {
SessionDisposition::Resultless { failure } => Some(format!(
"{} ({}{}{})",
session.device_id,
session.session_id,
failure.as_ref().map(|_| "; ").unwrap_or_default(),
failure.as_deref().unwrap_or_default()
)),
_ => None,
})
.collect::<Vec<_>>();
let mut groups = Vec::new();
if !missing.is_empty() {
groups.push(format!(
"missing collected sessions: {}",
missing.join(", ")
));
}
if !non_passed.is_empty() {
groups.push(format!("non-passed sessions: {}", non_passed.join(", ")));
}
if !resultless.is_empty() {
groups.push(format!("result-less sessions: {}", resultless.join(", ")));
}
write!(formatter, "{}", groups.join("; "))
}
}
fn session_label(session: &SessionAssessment) -> String {
format!("{} ({})", session.device_id, session.session_id)
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum ReconcileError {
#[error("terminal provider build reported no device sessions")]
NoExpectedSessions,
#[error("duplicate expected session: {session_id}")]
DuplicateExpectedSession { session_id: String },
#[error("duplicate expected device: {device_id}")]
DuplicateExpectedDevice { device_id: String },
#[error("duplicate collected session: {session_id}")]
DuplicateCollectedSession { session_id: String },
#[error("unexpected collected session: {session_id}")]
UnexpectedCollectedSession { session_id: String },
#[error("duplicate benchmark results: {device_id} ({session_id}, got {report_count})")]
DuplicateReports {
device_id: String,
session_id: String,
report_count: usize,
},
}
pub fn reconcile_sessions(
expected: &[ExpectedSession],
collected: &[CollectedSession],
) -> Result<MatrixAssessment, ReconcileError> {
if expected.is_empty() {
return Err(ReconcileError::NoExpectedSessions);
}
let mut expected_session_ids = HashSet::with_capacity(expected.len());
let mut expected_device_ids = HashSet::with_capacity(expected.len());
for session in expected {
if !expected_session_ids.insert(session.session_id.as_str()) {
return Err(ReconcileError::DuplicateExpectedSession {
session_id: session.session_id.clone(),
});
}
if !expected_device_ids.insert(session.device_id.as_str()) {
return Err(ReconcileError::DuplicateExpectedDevice {
device_id: session.device_id.clone(),
});
}
}
let mut collected_by_session = HashMap::with_capacity(collected.len());
for session in collected {
if !expected_session_ids.contains(session.session_id.as_str()) {
return Err(ReconcileError::UnexpectedCollectedSession {
session_id: session.session_id.clone(),
});
}
if collected_by_session
.insert(session.session_id.as_str(), session)
.is_some()
{
return Err(ReconcileError::DuplicateCollectedSession {
session_id: session.session_id.clone(),
});
}
}
let mut completed = 0usize;
let mut sessions = Vec::with_capacity(expected.len());
for session in expected {
let disposition = match collected_by_session.get(session.session_id.as_str()) {
None => SessionDisposition::Missing,
Some(collected) if collected.report_count > 1 => {
return Err(ReconcileError::DuplicateReports {
device_id: session.device_id.clone(),
session_id: session.session_id.clone(),
report_count: collected.report_count,
});
}
Some(collected) if !session.status.eq_ignore_ascii_case("passed") => {
SessionDisposition::NonPassed {
status: session.status.clone(),
failure: collected.failure.clone(),
}
}
Some(collected) if collected.report_count == 0 => SessionDisposition::Resultless {
failure: collected.failure.clone(),
},
Some(_) => {
completed += 1;
SessionDisposition::Complete
}
};
sessions.push(SessionAssessment {
session_id: session.session_id.clone(),
device_id: session.device_id.clone(),
disposition,
});
}
let outcome = if completed == expected.len() {
MatrixOutcome::Complete
} else if completed == 0 {
MatrixOutcome::Failed
} else {
MatrixOutcome::Partial
};
Ok(MatrixAssessment { outcome, sessions })
}
#[cfg(test)]
mod tests {
use super::*;
fn expected(count: usize) -> Vec<ExpectedSession> {
(0..count)
.map(|index| ExpectedSession {
session_id: format!("session-{index}"),
device_id: format!("device-{index}"),
status: "passed".to_owned(),
})
.collect()
}
fn collected(count: usize) -> Vec<CollectedSession> {
(0..count)
.rev()
.map(|index| CollectedSession {
session_id: format!("session-{index}"),
report_count: 1,
failure: None,
})
.collect()
}
#[test]
fn complete_matrices_are_deterministic_at_one_ten_and_fifty_devices() {
for count in [1, 10, 50] {
let expected = expected(count);
let assessment = reconcile_sessions(&expected, &collected(count)).unwrap();
assert_eq!(assessment.outcome(), MatrixOutcome::Complete);
assert!(assessment.is_complete());
assert_eq!(assessment.sessions().len(), count);
for (index, session) in assessment.sessions().iter().enumerate() {
assert_eq!(session.session_id, format!("session-{index}"));
assert_eq!(session.disposition, SessionDisposition::Complete);
}
}
}
#[test]
fn one_of_ten_is_partial_and_never_complete() {
let assessment = reconcile_sessions(&expected(10), &collected(1)).unwrap();
assert_eq!(assessment.outcome(), MatrixOutcome::Partial);
assert!(!assessment.is_complete());
assert!(
assessment
.to_string()
.contains("missing collected sessions")
);
assert!(assessment.to_string().contains("session-9"));
}
#[test]
fn zero_successful_sessions_is_failed() {
let mut expected = expected(1);
expected[0].status = "failed".to_owned();
let collected = [CollectedSession {
session_id: "session-0".to_owned(),
report_count: 0,
failure: Some("benchmark timed out".to_owned()),
}];
let assessment = reconcile_sessions(&expected, &collected).unwrap();
assert_eq!(assessment.outcome(), MatrixOutcome::Failed);
assert!(assessment.to_string().contains("non-passed sessions"));
assert!(assessment.to_string().contains("benchmark timed out"));
}
#[test]
fn passed_session_without_a_report_is_failed() {
let collected = [CollectedSession {
session_id: "session-0".to_owned(),
report_count: 0,
failure: Some("no report marker".to_owned()),
}];
let assessment = reconcile_sessions(&expected(1), &collected).unwrap();
assert_eq!(assessment.outcome(), MatrixOutcome::Failed);
assert!(assessment.to_string().contains("result-less sessions"));
}
#[test]
fn ambiguity_and_unexpected_sessions_fail_closed() {
let mut duplicate_expected = expected(2);
duplicate_expected[1].device_id = duplicate_expected[0].device_id.clone();
assert!(matches!(
reconcile_sessions(&duplicate_expected, &collected(2)),
Err(ReconcileError::DuplicateExpectedDevice { .. })
));
let duplicate_collected = [collected(1)[0].clone(), collected(1)[0].clone()];
assert!(matches!(
reconcile_sessions(&expected(1), &duplicate_collected),
Err(ReconcileError::DuplicateCollectedSession { .. })
));
let unexpected = [CollectedSession {
session_id: "not-requested".to_owned(),
report_count: 1,
failure: None,
}];
assert!(matches!(
reconcile_sessions(&expected(1), &unexpected),
Err(ReconcileError::UnexpectedCollectedSession { .. })
));
}
#[test]
fn multiple_reports_for_one_session_are_ambiguous() {
let collected = [CollectedSession {
session_id: "session-0".to_owned(),
report_count: 2,
failure: None,
}];
assert_eq!(
reconcile_sessions(&expected(1), &collected),
Err(ReconcileError::DuplicateReports {
device_id: "device-0".to_owned(),
session_id: "session-0".to_owned(),
report_count: 2,
})
);
}
}