use std::collections::HashMap;
use std::fmt;
use std::fs;
use std::path::Path;
#[derive(Debug, Clone, PartialEq)]
pub struct DebugRegressionSample {
pub sample_index: u64,
pub fps: f32,
pub selected_tiles: usize,
pub loaded_tiles: usize,
pub pending_tiles: usize,
pub visible_tiles: usize,
pub exact_visible_tiles: usize,
pub fallback_visible_tiles: usize,
pub missing_visible_tiles: usize,
pub cache_renderable_entries: usize,
pub queued_requests: usize,
pub in_flight_requests: usize,
pub max_concurrent_requests: usize,
pub counter_cancelled_stale_pending: u64,
}
impl DebugRegressionSample {
fn visible_coverage_ratio(&self) -> f64 {
if self.visible_tiles == 0 {
return 0.0;
}
(self.exact_visible_tiles + self.fallback_visible_tiles) as f64 / self.visible_tiles as f64
}
fn exact_coverage_ratio(&self) -> f64 {
if self.visible_tiles == 0 {
return 0.0;
}
self.exact_visible_tiles as f64 / self.visible_tiles as f64
}
fn is_fallback_only(&self) -> bool {
self.visible_tiles > 0 && self.exact_visible_tiles == 0 && self.fallback_visible_tiles > 0
}
fn is_saturated(&self) -> bool {
self.max_concurrent_requests > 0 && self.in_flight_requests >= self.max_concurrent_requests
}
}
#[derive(Debug, Clone, PartialEq, Default)]
pub struct DebugRegressionMetrics {
pub total_samples: usize,
pub first_loaded_sample_index: Option<u64>,
pub first_renderable_sample_index: Option<u64>,
pub first_majority_visible_coverage_sample_index: Option<u64>,
pub first_majority_exact_coverage_sample_index: Option<u64>,
pub longest_fallback_only_streak: usize,
pub longest_missing_visible_streak: usize,
pub longest_saturated_request_streak: usize,
pub max_missing_visible_tiles: usize,
pub max_queued_requests: usize,
pub min_fps: f32,
pub final_counter_cancelled_stale_pending: u64,
pub final_exact_coverage_ratio: f64,
}
impl DebugRegressionMetrics {
pub fn to_report(&self) -> String {
let mut lines = Vec::new();
lines.push(format!("samples: {}", self.total_samples));
lines.push(format!(
"first-loaded-sample: {}",
option_or_na(self.first_loaded_sample_index)
));
lines.push(format!(
"first-renderable-sample: {}",
option_or_na(self.first_renderable_sample_index)
));
lines.push(format!(
"first-majority-visible-coverage-sample: {}",
option_or_na(self.first_majority_visible_coverage_sample_index)
));
lines.push(format!(
"first-majority-exact-coverage-sample: {}",
option_or_na(self.first_majority_exact_coverage_sample_index)
));
lines.push(format!(
"longest-fallback-only-streak: {}",
self.longest_fallback_only_streak
));
lines.push(format!(
"longest-missing-visible-streak: {}",
self.longest_missing_visible_streak
));
lines.push(format!(
"longest-saturated-request-streak: {}",
self.longest_saturated_request_streak
));
lines.push(format!(
"max-missing-visible-tiles: {}",
self.max_missing_visible_tiles
));
lines.push(format!("max-queued-requests: {}", self.max_queued_requests));
lines.push(format!("min-fps: {:.3}", self.min_fps));
lines.push(format!(
"final-stale-cancels: {}",
self.final_counter_cancelled_stale_pending
));
lines.push(format!(
"final-exact-coverage-ratio: {:.3}",
self.final_exact_coverage_ratio
));
lines.join("\n")
}
}
#[derive(Debug, Clone, PartialEq, Default)]
pub struct DebugRegressionThresholds {
pub max_first_loaded_sample_index: Option<u64>,
pub max_first_renderable_sample_index: Option<u64>,
pub max_first_majority_visible_coverage_sample_index: Option<u64>,
pub max_first_majority_exact_coverage_sample_index: Option<u64>,
pub max_longest_fallback_only_streak: Option<usize>,
pub max_longest_missing_visible_streak: Option<usize>,
pub max_longest_saturated_request_streak: Option<usize>,
pub max_missing_visible_tiles: Option<usize>,
pub max_final_counter_cancelled_stale_pending: Option<u64>,
pub min_final_exact_coverage_ratio: Option<f64>,
}
impl DebugRegressionThresholds {
pub fn is_empty(&self) -> bool {
self.max_first_loaded_sample_index.is_none()
&& self.max_first_renderable_sample_index.is_none()
&& self
.max_first_majority_visible_coverage_sample_index
.is_none()
&& self
.max_first_majority_exact_coverage_sample_index
.is_none()
&& self.max_longest_fallback_only_streak.is_none()
&& self.max_longest_missing_visible_streak.is_none()
&& self.max_longest_saturated_request_streak.is_none()
&& self.max_missing_visible_tiles.is_none()
&& self.max_final_counter_cancelled_stale_pending.is_none()
&& self.min_final_exact_coverage_ratio.is_none()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DebugRegressionFailure {
pub metric: &'static str,
pub message: String,
}
#[derive(Debug)]
pub enum DebugRegressionError {
Io(std::io::Error),
MissingHeader(&'static str),
EmptyInput,
InvalidRecord {
line: usize,
reason: String,
},
}
impl fmt::Display for DebugRegressionError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Io(err) => write!(f, "failed to read debug regression CSV: {err}"),
Self::MissingHeader(name) => write!(
f,
"debug regression CSV is missing required header `{name}`"
),
Self::EmptyInput => write!(f, "debug regression CSV is empty"),
Self::InvalidRecord { line, reason } => {
write!(
f,
"invalid debug regression CSV record at line {line}: {reason}"
)
}
}
}
}
impl std::error::Error for DebugRegressionError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Io(err) => Some(err),
_ => None,
}
}
}
impl From<std::io::Error> for DebugRegressionError {
fn from(value: std::io::Error) -> Self {
Self::Io(value)
}
}
pub fn load_debug_regression_samples(
path: impl AsRef<Path>,
) -> Result<Vec<DebugRegressionSample>, DebugRegressionError> {
let csv = fs::read_to_string(path)?;
parse_debug_regression_csv(&csv)
}
pub fn parse_debug_regression_csv(
csv: &str,
) -> Result<Vec<DebugRegressionSample>, DebugRegressionError> {
let mut lines = csv
.lines()
.enumerate()
.filter(|(_, line)| !line.trim().is_empty());
let Some((_, header_line)) = lines.next() else {
return Err(DebugRegressionError::EmptyInput);
};
let header_fields = parse_csv_line(header_line)
.map_err(|reason| DebugRegressionError::InvalidRecord { line: 1, reason })?;
let header_map = header_index_map(&header_fields);
let mut samples = Vec::new();
for (index, line) in lines {
let fields =
parse_csv_line(line).map_err(|reason| DebugRegressionError::InvalidRecord {
line: index + 1,
reason,
})?;
samples.push(DebugRegressionSample {
sample_index: parse_required_u64(&fields, &header_map, "sample_index", index + 1)?,
fps: parse_required_f32(&fields, &header_map, "fps", index + 1)?,
selected_tiles: parse_required_usize(
&fields,
&header_map,
"selected_tiles",
index + 1,
)?,
loaded_tiles: parse_required_usize(&fields, &header_map, "loaded_tiles", index + 1)?,
pending_tiles: parse_required_usize(&fields, &header_map, "pending_tiles", index + 1)?,
visible_tiles: parse_required_usize(&fields, &header_map, "visible_tiles", index + 1)?,
exact_visible_tiles: parse_required_usize(
&fields,
&header_map,
"exact_visible_tiles",
index + 1,
)?,
fallback_visible_tiles: parse_required_usize(
&fields,
&header_map,
"fallback_visible_tiles",
index + 1,
)?,
missing_visible_tiles: parse_required_usize(
&fields,
&header_map,
"missing_visible_tiles",
index + 1,
)?,
cache_renderable_entries: parse_required_usize(
&fields,
&header_map,
"cache_renderable_entries",
index + 1,
)?,
queued_requests: parse_required_usize(
&fields,
&header_map,
"queued_requests",
index + 1,
)?,
in_flight_requests: parse_required_usize(
&fields,
&header_map,
"in_flight_requests",
index + 1,
)?,
max_concurrent_requests: parse_required_usize(
&fields,
&header_map,
"max_concurrent_requests",
index + 1,
)?,
counter_cancelled_stale_pending: parse_required_u64(
&fields,
&header_map,
"counter_cancelled_stale_pending",
index + 1,
)?,
});
}
Ok(samples)
}
pub fn compute_debug_regression_metrics(
samples: &[DebugRegressionSample],
) -> DebugRegressionMetrics {
if samples.is_empty() {
return DebugRegressionMetrics::default();
}
let mut metrics = DebugRegressionMetrics {
total_samples: samples.len(),
first_loaded_sample_index: samples
.iter()
.find(|sample| sample.loaded_tiles > 0)
.map(|sample| sample.sample_index),
first_renderable_sample_index: samples
.iter()
.find(|sample| sample.cache_renderable_entries > 0)
.map(|sample| sample.sample_index),
first_majority_visible_coverage_sample_index: samples
.iter()
.find(|sample| sample.visible_coverage_ratio() >= 0.5)
.map(|sample| sample.sample_index),
first_majority_exact_coverage_sample_index: samples
.iter()
.find(|sample| sample.exact_coverage_ratio() >= 0.5)
.map(|sample| sample.sample_index),
min_fps: f32::MAX,
..DebugRegressionMetrics::default()
};
let mut fallback_streak = 0usize;
let mut missing_streak = 0usize;
let mut saturated_streak = 0usize;
for sample in samples {
metrics.max_missing_visible_tiles = metrics
.max_missing_visible_tiles
.max(sample.missing_visible_tiles);
metrics.max_queued_requests = metrics.max_queued_requests.max(sample.queued_requests);
metrics.min_fps = metrics.min_fps.min(sample.fps);
if sample.is_fallback_only() {
fallback_streak += 1;
metrics.longest_fallback_only_streak =
metrics.longest_fallback_only_streak.max(fallback_streak);
} else {
fallback_streak = 0;
}
if sample.missing_visible_tiles > 0 {
missing_streak += 1;
metrics.longest_missing_visible_streak =
metrics.longest_missing_visible_streak.max(missing_streak);
} else {
missing_streak = 0;
}
if sample.is_saturated() {
saturated_streak += 1;
metrics.longest_saturated_request_streak = metrics
.longest_saturated_request_streak
.max(saturated_streak);
} else {
saturated_streak = 0;
}
}
if let Some(last) = samples.last() {
metrics.final_counter_cancelled_stale_pending = last.counter_cancelled_stale_pending;
metrics.final_exact_coverage_ratio = last.exact_coverage_ratio();
}
metrics
}
pub fn evaluate_debug_regression_thresholds(
metrics: &DebugRegressionMetrics,
thresholds: &DebugRegressionThresholds,
) -> Vec<DebugRegressionFailure> {
let mut failures = Vec::new();
push_max_optional_failure(
&mut failures,
"first_loaded_sample_index",
thresholds.max_first_loaded_sample_index,
metrics.first_loaded_sample_index,
);
push_max_optional_failure(
&mut failures,
"first_renderable_sample_index",
thresholds.max_first_renderable_sample_index,
metrics.first_renderable_sample_index,
);
push_max_optional_failure(
&mut failures,
"first_majority_visible_coverage_sample_index",
thresholds.max_first_majority_visible_coverage_sample_index,
metrics.first_majority_visible_coverage_sample_index,
);
push_max_optional_failure(
&mut failures,
"first_majority_exact_coverage_sample_index",
thresholds.max_first_majority_exact_coverage_sample_index,
metrics.first_majority_exact_coverage_sample_index,
);
push_max_failure(
&mut failures,
"longest_fallback_only_streak",
thresholds.max_longest_fallback_only_streak,
metrics.longest_fallback_only_streak,
);
push_max_failure(
&mut failures,
"longest_missing_visible_streak",
thresholds.max_longest_missing_visible_streak,
metrics.longest_missing_visible_streak,
);
push_max_failure(
&mut failures,
"longest_saturated_request_streak",
thresholds.max_longest_saturated_request_streak,
metrics.longest_saturated_request_streak,
);
push_max_failure(
&mut failures,
"max_missing_visible_tiles",
thresholds.max_missing_visible_tiles,
metrics.max_missing_visible_tiles,
);
push_max_failure(
&mut failures,
"final_counter_cancelled_stale_pending",
thresholds.max_final_counter_cancelled_stale_pending,
metrics.final_counter_cancelled_stale_pending,
);
if let Some(minimum) = thresholds.min_final_exact_coverage_ratio {
if metrics.final_exact_coverage_ratio < minimum {
failures.push(DebugRegressionFailure {
metric: "final_exact_coverage_ratio",
message: format!(
"expected final_exact_coverage_ratio >= {:.3}, got {:.3}",
minimum, metrics.final_exact_coverage_ratio
),
});
}
}
failures
}
fn option_or_na<T: fmt::Display>(value: Option<T>) -> String {
match value {
Some(value) => value.to_string(),
None => "n/a".to_string(),
}
}
fn push_max_optional_failure<T>(
failures: &mut Vec<DebugRegressionFailure>,
metric: &'static str,
maximum: Option<T>,
actual: Option<T>,
) where
T: fmt::Display + PartialOrd + Copy,
{
if let Some(maximum) = maximum {
match actual {
Some(actual) if actual > maximum => failures.push(DebugRegressionFailure {
metric,
message: format!("expected {metric} <= {maximum}, got {actual}"),
}),
None => failures.push(DebugRegressionFailure {
metric,
message: format!("expected {metric} <= {maximum}, got n/a"),
}),
_ => {}
}
}
}
fn push_max_failure<T>(
failures: &mut Vec<DebugRegressionFailure>,
metric: &'static str,
maximum: Option<T>,
actual: T,
) where
T: fmt::Display + PartialOrd + Copy,
{
if let Some(maximum) = maximum {
if actual > maximum {
failures.push(DebugRegressionFailure {
metric,
message: format!("expected {metric} <= {maximum}, got {actual}"),
});
}
}
}
fn header_index_map(headers: &[String]) -> HashMap<&str, usize> {
headers
.iter()
.enumerate()
.map(|(index, value)| (value.as_str(), index))
.collect()
}
fn parse_required_usize(
fields: &[String],
headers: &HashMap<&str, usize>,
header: &'static str,
line: usize,
) -> Result<usize, DebugRegressionError> {
let raw = parse_required_field(fields, headers, header)?;
raw.parse::<usize>()
.map_err(|_| DebugRegressionError::InvalidRecord {
line,
reason: format!("field `{header}` is not a valid usize: `{raw}`"),
})
}
fn parse_required_u64(
fields: &[String],
headers: &HashMap<&str, usize>,
header: &'static str,
line: usize,
) -> Result<u64, DebugRegressionError> {
let raw = parse_required_field(fields, headers, header)?;
raw.parse::<u64>()
.map_err(|_| DebugRegressionError::InvalidRecord {
line,
reason: format!("field `{header}` is not a valid u64: `{raw}`"),
})
}
fn parse_required_f32(
fields: &[String],
headers: &HashMap<&str, usize>,
header: &'static str,
line: usize,
) -> Result<f32, DebugRegressionError> {
let raw = parse_required_field(fields, headers, header)?;
raw.parse::<f32>()
.map_err(|_| DebugRegressionError::InvalidRecord {
line,
reason: format!("field `{header}` is not a valid f32: `{raw}`"),
})
}
fn parse_required_field<'a>(
fields: &'a [String],
headers: &HashMap<&str, usize>,
header: &'static str,
) -> Result<&'a str, DebugRegressionError> {
let Some(index) = headers.get(header).copied() else {
return Err(DebugRegressionError::MissingHeader(header));
};
let Some(value) = fields.get(index) else {
return Err(DebugRegressionError::InvalidRecord {
line: 0,
reason: format!("row does not contain field `{header}`"),
});
};
Ok(value.as_str())
}
fn parse_csv_line(line: &str) -> Result<Vec<String>, String> {
let mut fields = Vec::new();
let mut current = String::new();
let mut chars = line.chars().peekable();
let mut in_quotes = false;
while let Some(ch) = chars.next() {
match ch {
'"' => {
if in_quotes {
if matches!(chars.peek(), Some('"')) {
current.push('"');
chars.next();
} else {
in_quotes = false;
}
} else if current.is_empty() {
in_quotes = true;
} else {
return Err("unexpected quote in unquoted field".to_string());
}
}
',' if !in_quotes => {
fields.push(std::mem::take(&mut current));
}
_ => current.push(ch),
}
}
if in_quotes {
return Err("unterminated quoted field".to_string());
}
fields.push(current);
Ok(fields)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_debug_csv_handles_quoted_fields() {
let csv = "sample_index,sample_image,fps,selected_tiles,loaded_tiles,pending_tiles,visible_tiles,exact_visible_tiles,fallback_visible_tiles,missing_visible_tiles,cache_renderable_entries,queued_requests,in_flight_requests,max_concurrent_requests,counter_cancelled_stale_pending\n1,\"docs/debug/sample_0001.jpg\",9.143,82,82,0,82,42,0,0,207,0,8,32,0\n";
let samples = parse_debug_regression_csv(csv).unwrap();
assert_eq!(samples.len(), 1);
assert_eq!(samples[0].sample_index, 1);
assert_eq!(samples[0].fps, 9.143);
assert_eq!(samples[0].exact_visible_tiles, 42);
assert_eq!(samples[0].cache_renderable_entries, 207);
}
#[test]
fn compute_metrics_tracks_streaks_and_thresholds() {
let csv = "sample_index,sample_image,fps,selected_tiles,loaded_tiles,pending_tiles,visible_tiles,exact_visible_tiles,fallback_visible_tiles,missing_visible_tiles,cache_renderable_entries,queued_requests,in_flight_requests,max_concurrent_requests,counter_cancelled_stale_pending\n1,\"a\",10.0,10,0,10,10,0,10,0,0,5,4,4,0\n2,\"b\",9.0,10,0,10,10,0,10,2,0,7,4,4,2\n3,\"c\",8.0,10,8,2,10,6,2,0,8,2,2,4,3\n4,\"d\",7.0,10,10,0,10,8,2,0,8,1,1,4,4\n";
let samples = parse_debug_regression_csv(csv).unwrap();
let metrics = compute_debug_regression_metrics(&samples);
assert_eq!(metrics.total_samples, 4);
assert_eq!(metrics.first_loaded_sample_index, Some(3));
assert_eq!(metrics.first_renderable_sample_index, Some(3));
assert_eq!(
metrics.first_majority_visible_coverage_sample_index,
Some(1)
);
assert_eq!(metrics.first_majority_exact_coverage_sample_index, Some(3));
assert_eq!(metrics.longest_fallback_only_streak, 2);
assert_eq!(metrics.longest_missing_visible_streak, 1);
assert_eq!(metrics.longest_saturated_request_streak, 2);
assert_eq!(metrics.max_missing_visible_tiles, 2);
assert_eq!(metrics.final_counter_cancelled_stale_pending, 4);
let thresholds = DebugRegressionThresholds {
max_first_majority_exact_coverage_sample_index: Some(2),
max_longest_fallback_only_streak: Some(1),
max_missing_visible_tiles: Some(1),
max_final_counter_cancelled_stale_pending: Some(3),
min_final_exact_coverage_ratio: Some(0.9),
..DebugRegressionThresholds::default()
};
let failures = evaluate_debug_regression_thresholds(&metrics, &thresholds);
assert_eq!(failures.len(), 5);
}
#[test]
fn workspace_capture_structural_invariants() {
let csv = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../docs/debug/rustial_debug_values.csv"
));
let samples = parse_debug_regression_csv(csv).unwrap();
let metrics = compute_debug_regression_metrics(&samples);
assert!(metrics.total_samples >= 300);
assert_eq!(metrics.first_loaded_sample_index, Some(1));
assert_eq!(metrics.first_renderable_sample_index, Some(1));
}
}