use crate::profile::apply::{ApplyOutcome, ApplyStatus};
use crate::profile::SettingValue;
use serde::{Deserialize, Serialize};
use std::time::Duration;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Direction {
HigherIsBetter,
LowerIsBetter,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MetricSpec {
pub id: String,
pub direction: Direction,
pub min_effect: f64,
pub requires_load: bool,
}
pub trait MetricSource: Send + Sync {
fn spec(&self) -> MetricSpec;
fn sample(&self) -> Option<f64>;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SamplingPlan {
pub count: usize,
pub gap: Duration,
}
impl Default for SamplingPlan {
fn default() -> Self {
Self {
count: 8,
gap: Duration::from_millis(250),
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Measurement {
pub samples: Vec<f64>,
pub median: f64,
pub mad: f64,
pub missed: usize,
}
impl Measurement {
pub fn from_samples(samples: Vec<f64>, missed: usize) -> Option<Self> {
if samples.is_empty() {
return None;
}
let median = median_of(&samples);
let deviations: Vec<f64> = samples.iter().map(|s| (s - median).abs()).collect();
let mad = median_of(&deviations);
Some(Self {
samples,
median,
mad,
missed,
})
}
}
fn median_of(values: &[f64]) -> f64 {
let mut sorted = values.to_vec();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let n = sorted.len();
if n == 0 {
return 0.0;
}
if n.is_multiple_of(2) {
(sorted[n / 2 - 1] + sorted[n / 2]) / 2.0
} else {
sorted[n / 2]
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "verdict")]
pub enum Verdict {
Improved {
metric: String,
before: f64,
after: f64,
delta: f64,
},
Unchanged {
metric: String,
before: f64,
after: f64,
delta: f64,
threshold: f64,
},
Regressed {
metric: String,
before: f64,
after: f64,
delta: f64,
},
Unverifiable { reason: String },
}
impl Verdict {
pub fn warrants_revert(&self) -> bool {
matches!(self, Verdict::Regressed { .. })
}
}
pub fn measure(source: &dyn MetricSource, plan: &SamplingPlan) -> Option<Measurement> {
let mut samples = Vec::with_capacity(plan.count);
let mut missed = 0usize;
for i in 0..plan.count {
match source.sample() {
Some(v) => samples.push(v),
None => missed += 1,
}
if i + 1 < plan.count {
std::thread::sleep(plan.gap);
}
}
Measurement::from_samples(samples, missed)
}
pub fn compare(spec: &MetricSpec, before: &Measurement, after: &Measurement) -> Verdict {
for (window, label) in [(before, "before"), (after, "after")] {
let succeeded = window.samples.len();
let attempted = succeeded + window.missed;
if succeeded * 2 <= attempted {
return Verdict::Unverifiable {
reason: format!(
"{} is inconclusive: only {succeeded} of {attempted} readings {label} \
the write succeeded, which is not a majority",
spec.id
),
};
}
}
let delta = after.median - before.median;
let noise = before.mad + after.mad;
let threshold = spec.min_effect.max(noise);
if delta.abs() < threshold {
return Verdict::Unchanged {
metric: spec.id.clone(),
before: before.median,
after: after.median,
delta,
threshold,
};
}
let better = match spec.direction {
Direction::HigherIsBetter => delta > 0.0,
Direction::LowerIsBetter => delta < 0.0,
};
if better {
Verdict::Improved {
metric: spec.id.clone(),
before: before.median,
after: after.median,
delta,
}
} else {
Verdict::Regressed {
metric: spec.id.clone(),
before: before.median,
after: after.median,
delta,
}
}
}
pub fn metric_for(setting_id: &str) -> Option<Box<dyn MetricSource>> {
let _ = setting_id;
None
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct VerifiedOutcome {
pub outcome: ApplyOutcome,
pub verdict: Verdict,
pub reverted: Option<ApplyOutcome>,
}
pub fn apply_verified(
setting_id: &str,
value: SettingValue,
confirm: bool,
settle: Duration,
plan: &SamplingPlan,
) -> VerifiedOutcome {
let Some(source) = metric_for(setting_id) else {
let outcome = crate::profile::apply::apply_setting_reversible(setting_id, value, confirm);
return VerifiedOutcome {
outcome,
verdict: Verdict::Unverifiable {
reason: format!("no metric is declared for {setting_id}"),
},
reverted: None,
};
};
let spec = source.spec();
if spec.requires_load {
let reason = match machine_is_loaded() {
Some(true) => None,
Some(false) => Some(format!(
"{} is only meaningful under load and the machine is idle",
spec.id
)),
None => Some(format!(
concat!(
"{} is only meaningful under load, and CPU utilisation could not ",
"be read to tell whether the machine is loaded"
),
spec.id
)),
};
if let Some(reason) = reason {
let outcome =
crate::profile::apply::apply_setting_reversible(setting_id, value, confirm);
return VerifiedOutcome {
outcome,
verdict: Verdict::Unverifiable { reason },
reverted: None,
};
}
}
verify_around(
source.as_ref(),
settle,
plan,
|| crate::profile::apply::apply_setting_reversible(setting_id, value, confirm),
|o| crate::profile::apply::revert_setting(o, confirm),
)
}
fn verify_around<A, R>(
source: &dyn MetricSource,
settle: Duration,
plan: &SamplingPlan,
apply: A,
revert: R,
) -> VerifiedOutcome
where
A: FnOnce() -> ApplyOutcome,
R: FnOnce(&ApplyOutcome) -> ApplyOutcome,
{
let spec = source.spec();
let before = measure(source, plan);
let outcome = apply();
if outcome.status != ApplyStatus::Applied {
return VerifiedOutcome {
outcome,
verdict: Verdict::Unverifiable {
reason: "the write did not take effect, so there is nothing to measure".to_string(),
},
reverted: None,
};
}
let Some(before) = before else {
return VerifiedOutcome {
outcome,
verdict: Verdict::Unverifiable {
reason: format!("{} could not be read before the write", spec.id),
},
reverted: None,
};
};
std::thread::sleep(settle);
let Some(after) = measure(source, plan) else {
return VerifiedOutcome {
outcome,
verdict: Verdict::Unverifiable {
reason: format!("{} could not be read after the write", spec.id),
},
reverted: None,
};
};
let verdict = compare(&spec, &before, &after);
let reverted = if verdict.warrants_revert() {
Some(revert(&outcome))
} else {
None
};
VerifiedOutcome {
outcome,
verdict,
reverted,
}
}
fn machine_is_loaded() -> Option<bool> {
const BUSY_PERCENT: f32 = 20.0;
crate::tuning::serve::read_cpu_utilization().map(|u| u >= BUSY_PERCENT)
}
#[cfg(test)]
mod tests {
use super::*;
struct Scripted {
spec: MetricSpec,
readings: std::sync::Mutex<Vec<Option<f64>>>,
}
impl Scripted {
fn new(direction: Direction, min_effect: f64, readings: Vec<Option<f64>>) -> Self {
Self {
spec: MetricSpec {
id: "scripted".to_string(),
direction,
min_effect,
requires_load: false,
},
readings: std::sync::Mutex::new(readings),
}
}
}
impl MetricSource for Scripted {
fn spec(&self) -> MetricSpec {
self.spec.clone()
}
fn sample(&self) -> Option<f64> {
let mut r = self.readings.lock().unwrap();
if r.is_empty() {
None
} else {
r.remove(0)
}
}
}
fn spec(direction: Direction, min_effect: f64) -> MetricSpec {
MetricSpec {
id: "m".to_string(),
direction,
min_effect,
requires_load: false,
}
}
fn measurement(values: &[f64]) -> Measurement {
Measurement::from_samples(values.to_vec(), 0).expect("non-empty")
}
#[test]
fn a_clear_move_the_good_way_is_an_improvement() {
let before = measurement(&[10.0, 10.0, 10.0, 10.0]);
let after = measurement(&[20.0, 20.0, 20.0, 20.0]);
let v = compare(&spec(Direction::HigherIsBetter, 1.0), &before, &after);
assert!(matches!(v, Verdict::Improved { .. }), "got {v:?}");
assert!(!v.warrants_revert());
}
#[test]
fn the_same_move_is_a_regression_when_lower_is_better() {
let before = measurement(&[10.0, 10.0, 10.0, 10.0]);
let after = measurement(&[20.0, 20.0, 20.0, 20.0]);
let v = compare(&spec(Direction::LowerIsBetter, 1.0), &before, &after);
assert!(matches!(v, Verdict::Regressed { .. }), "got {v:?}");
assert!(
v.warrants_revert(),
"a demonstrated regression is the one case that must come back off"
);
}
#[test]
fn a_move_smaller_than_the_declared_minimum_effect_is_not_a_result() {
let before = measurement(&[10.0, 10.0, 10.0, 10.0]);
let after = measurement(&[10.4, 10.4, 10.4, 10.4]);
let v = compare(&spec(Direction::HigherIsBetter, 1.0), &before, &after);
match v {
Verdict::Unchanged { threshold, .. } => assert_eq!(threshold, 1.0),
other => panic!("expected Unchanged, got {other:?}"),
}
}
#[test]
fn a_noisy_metric_raises_its_own_bar() {
let steady_before = measurement(&[10.0, 10.0, 10.0, 10.0]);
let steady_after = measurement(&[15.0, 15.0, 15.0, 15.0]);
let steady = compare(
&spec(Direction::HigherIsBetter, 1.0),
&steady_before,
&steady_after,
);
assert!(
matches!(steady, Verdict::Improved { .. }),
"a clean 5-unit move should read as improvement, got {steady:?}"
);
let noisy_before = measurement(&[0.0, 10.0, 10.0, 20.0]);
let noisy_after = measurement(&[5.0, 15.0, 15.0, 25.0]);
let noisy = compare(
&spec(Direction::HigherIsBetter, 1.0),
&noisy_before,
&noisy_after,
);
assert!(
matches!(noisy, Verdict::Unchanged { .. }),
"the same 5-unit move inside 10 units of scatter is not evidence, got {noisy:?}"
);
}
#[test]
fn a_window_that_mostly_failed_to_read_gives_no_verdict() {
let before = measurement(&[10.0, 10.0, 10.0, 10.0, 10.0]);
let after = Measurement::from_samples(vec![50.0, 50.0, 50.0], 5).expect("non-empty");
let v = compare(&spec(Direction::HigherIsBetter, 1.0), &before, &after);
assert!(
matches!(v, Verdict::Unverifiable { .. }),
"3 of 8 readings must not decide an improvement, got {v:?}"
);
let after =
Measurement::from_samples(vec![50.0, 50.0, 50.0, 50.0, 50.0], 3).expect("non-empty");
let v = compare(&spec(Direction::HigherIsBetter, 1.0), &before, &after);
assert!(
matches!(v, Verdict::Improved { .. }),
"5 of 8 readings is a majority and must still count, got {v:?}"
);
}
#[test]
fn a_metric_that_never_reads_yields_no_measurement() {
let source = Scripted::new(Direction::HigherIsBetter, 1.0, vec![None, None, None]);
let plan = SamplingPlan {
count: 3,
gap: Duration::from_millis(0),
};
assert!(
measure(&source, &plan).is_none(),
"no readings must give no measurement, not a measurement of zero"
);
}
#[test]
fn failed_readings_are_counted_not_averaged_in() {
let source = Scripted::new(
Direction::HigherIsBetter,
1.0,
vec![Some(10.0), None, Some(10.0)],
);
let plan = SamplingPlan {
count: 3,
gap: Duration::from_millis(0),
};
let m = measure(&source, &plan).expect("two readings succeeded");
assert_eq!(m.samples, vec![10.0, 10.0]);
assert_eq!(m.missed, 1);
assert_eq!(
m.median, 10.0,
"a missed reading must not drag the median toward zero"
);
}
#[test]
fn unverifiable_and_unchanged_are_different_facts_once_serialised() {
let unchanged = Verdict::Unchanged {
metric: "m".into(),
before: 10.0,
after: 10.0,
delta: 0.0,
threshold: 1.0,
};
let unverifiable = Verdict::Unverifiable {
reason: "no metric".into(),
};
let a = serde_json::to_string(&unchanged).unwrap();
let b = serde_json::to_string(&unverifiable).unwrap();
assert_ne!(a, b);
assert!(a.contains("\"verdict\":\"unchanged\""), "{a}");
assert!(b.contains("\"verdict\":\"unverifiable\""), "{b}");
assert!(
!unverifiable.warrants_revert(),
"reverting on 'I could not tell' would undo nearly every write this \
crate makes, which is not the safe choice it looks like"
);
}
#[test]
fn no_setting_claims_a_metric_it_has_not_earned() {
for id in [
"active_scheme_guid",
"scaling_governor",
"persistence_mode",
"perf_level",
"gt_max_freq_mhz",
] {
assert!(
metric_for(id).is_none(),
"{id} declares a metric; the commit adding it must demonstrate \
that the metric moves when the setting changes"
);
}
}
fn fake_outcome(status: ApplyStatus) -> ApplyOutcome {
ApplyOutcome {
setting_id: "fake".into(),
subsystem: Some(crate::profile::Subsystem::Cpu),
requested: SettingValue::Text("new".into()),
status,
message: String::new(),
timestamp: 0,
previous: Some(SettingValue::Text("old".into())),
}
}
#[test]
fn a_regression_is_reverted() {
let reverted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let flag = reverted.clone();
let source = Scripted::new(
Direction::LowerIsBetter,
1.0,
vec![
Some(10.0),
Some(10.0),
Some(10.0),
Some(10.0),
Some(20.0),
Some(20.0),
Some(20.0),
Some(20.0),
],
);
let plan = SamplingPlan {
count: 4,
gap: Duration::from_millis(0),
};
let out = verify_around(
&source,
Duration::from_millis(0),
&plan,
|| fake_outcome(ApplyStatus::Applied),
|o| {
flag.store(true, std::sync::atomic::Ordering::SeqCst);
fake_outcome(o.status)
},
);
assert!(
matches!(out.verdict, Verdict::Regressed { .. }),
"got {:?}",
out.verdict
);
assert!(
reverted.load(std::sync::atomic::Ordering::SeqCst),
"a regression that is measured and then left in place is worse than \
not measuring at all"
);
assert!(out.reverted.is_some(), "the revert must be reported too");
}
#[test]
fn an_improvement_is_left_in_place() {
let reverted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let flag = reverted.clone();
let source = Scripted::new(
Direction::LowerIsBetter,
1.0,
vec![
Some(20.0),
Some(20.0),
Some(20.0),
Some(20.0),
Some(10.0),
Some(10.0),
Some(10.0),
Some(10.0),
],
);
let plan = SamplingPlan {
count: 4,
gap: Duration::from_millis(0),
};
let out = verify_around(
&source,
Duration::from_millis(0),
&plan,
|| fake_outcome(ApplyStatus::Applied),
|o| {
flag.store(true, std::sync::atomic::Ordering::SeqCst);
fake_outcome(o.status)
},
);
assert!(
matches!(out.verdict, Verdict::Improved { .. }),
"got {:?}",
out.verdict
);
assert!(
!reverted.load(std::sync::atomic::Ordering::SeqCst),
"nothing should have been undone"
);
}
#[test]
fn a_write_that_did_not_take_effect_is_not_measured() {
let source = Scripted::new(
Direction::HigherIsBetter,
1.0,
vec![Some(10.0), Some(10.0), Some(99.0), Some(99.0)],
);
let plan = SamplingPlan {
count: 2,
gap: Duration::from_millis(0),
};
let out = verify_around(
&source,
Duration::from_millis(0),
&plan,
|| fake_outcome(ApplyStatus::Refused),
|o| fake_outcome(o.status),
);
match out.verdict {
Verdict::Unverifiable { reason } => {
assert!(reason.contains("did not take effect"), "{reason}")
}
other => panic!("expected Unverifiable, got {other:?}"),
}
assert!(out.reverted.is_none());
}
#[test]
fn an_unreadable_before_window_yields_no_verdict() {
let source = Scripted::new(
Direction::HigherIsBetter,
1.0,
vec![None, None, Some(50.0), Some(50.0)],
);
let plan = SamplingPlan {
count: 2,
gap: Duration::from_millis(0),
};
let out = verify_around(
&source,
Duration::from_millis(0),
&plan,
|| fake_outcome(ApplyStatus::Applied),
|o| fake_outcome(o.status),
);
match out.verdict {
Verdict::Unverifiable { reason } => {
assert!(reason.contains("before the write"), "{reason}")
}
other => panic!("expected Unverifiable, got {other:?}"),
}
}
#[test]
fn a_setting_with_no_metric_is_still_applied_and_reported_honestly() {
let out = apply_verified(
"definitely_not_a_real_setting",
SettingValue::Bool(true),
false,
Duration::from_millis(0),
&SamplingPlan {
count: 1,
gap: Duration::from_millis(0),
},
);
match out.verdict {
Verdict::Unverifiable { reason } => {
assert!(reason.contains("no metric"), "{reason}");
}
other => panic!("expected Unverifiable, got {other:?}"),
}
assert!(
out.reverted.is_none(),
"nothing measurable happened, so nothing should be undone"
);
}
}