use keyhog_scanner::hw_probe::ScanBackend;
use serde::{Deserialize, Serialize};
mod match_identity;
mod timing;
pub(super) use match_identity::{
canonical_match_differences, canonical_match_digest, canonical_matches,
canonical_matches_equal_reference, differing_canonical_match_fields, CanonicalMatch,
};
#[cfg(test)]
pub(super) use timing::{paired_candidate_is_faster_95, ColdWarmStatisticalModel};
pub(super) use timing::{BackendTimingEvidence, TimingConfidenceInterval};
use super::workload::MeasurementShapeEvidence;
use super::{AUTOROUTE_ACCELERATOR_WARM_TRIALS, AUTOROUTE_CALIBRATION_TRIALS};
pub(super) const MAX_AUTOROUTE_MEASURED_POINTS: usize = 64;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct MeasuredRoute {
pub(super) backend: ScanBackend,
pub(super) phase2_plain_localizer: bool,
pub(super) phase2_keyword_localizer: bool,
pub(super) gpu_pipeline_depth: u8,
}
impl MeasuredRoute {
pub(super) fn execution_route(self) -> keyhog_scanner::ScanExecutionRoute {
keyhog_scanner::ScanExecutionRoute {
decode_backend: if self.backend.is_gpu() {
ScanBackend::CpuFallback
} else {
self.backend
},
phase2_plain_localizer: self.phase2_plain_localizer,
phase2_keyword_localizer: self.phase2_keyword_localizer,
gpu_pipeline_depth: self.gpu_pipeline_depth,
}
}
}
const fn backend_route_complexity(backend: ScanBackend) -> u8 {
match backend {
ScanBackend::CpuFallback => 0,
ScanBackend::SimdCpu => 1,
ScanBackend::GpuCuda => 2,
ScanBackend::GpuMetal => 3,
_ => 4,
}
}
fn paired_route_trials_are_faster(selected: &[u128], competitor: &[u128]) -> bool {
if selected.len() != competitor.len() || selected.is_empty() {
return false;
}
timing::paired_candidate_is_faster_95(selected, competitor)
}
fn selected_route_margin_ns(
selected: MeasuredRoute,
candidates: &[(MeasuredRoute, u128)],
) -> Option<u128> {
let selected_time = candidates.iter().find(|(route, _)| *route == selected)?.1;
candidates
.iter()
.filter(|(route, _)| *route != selected)
.map(|(_, timing_ns)| *timing_ns)
.min()
.map(|next_time| next_time.saturating_sub(selected_time))
}
fn accelerator_cold_warm_route_evidence(
timing: &BackendTimingEvidence,
) -> Option<(u128, BackendTimingEvidence, u128)> {
let model = timing.cold_warm_model()?;
if model.warm_trials_ns.len() != AUTOROUTE_ACCELERATOR_WARM_TRIALS {
return None;
}
let warm_timing = BackendTimingEvidence::from_trial_ns(model.warm_trials_ns.clone())?;
if !warm_timing.is_valid_for_trials(AUTOROUTE_ACCELERATOR_WARM_TRIALS) {
return None;
}
let route_ns = model.cold_one_shot_ns.max(model.warm_median_ns);
Some((model.cold_one_shot_ns, warm_timing, route_ns))
}
pub(super) fn gpu_cold_warm_route_evidence(
timing: &BackendTimingEvidence,
) -> Option<(u128, BackendTimingEvidence, u128)> {
accelerator_cold_warm_route_evidence(timing)
}
pub(super) fn simd_cold_warm_route_evidence(
timing: &BackendTimingEvidence,
) -> Option<(u128, BackendTimingEvidence, u128)> {
accelerator_cold_warm_route_evidence(timing)
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct BackendParityReceipt {
pub(super) backend: String,
pub(super) phase2_plain_localizer: bool,
pub(super) phase2_keyword_localizer: bool,
pub(super) gpu_pipeline_depth: u8,
pub(super) gpu_dispatch_capability: Option<String>,
pub(super) gpu_slot_input_capacity_bytes: Option<u64>,
pub(super) gpu_slot_match_capacity: Option<u32>,
pub(super) peer_identity: Option<String>,
pub(super) correctness_digest: u64,
pub(super) completed_trials: usize,
pub(super) evidence_digest: u64,
}
impl BackendParityReceipt {
fn new(
route: MeasuredRoute,
timing_entry: &RouteTimingEvidence,
correctness_digest: u64,
) -> Self {
let peer_identity = timing_entry.peer_identity.as_deref();
let timing = &timing_entry.timing;
let completed_trials = timing.trials_ns.len();
let evidence_digest = Self::evidence_digest_for(
route,
peer_identity,
correctness_digest,
completed_trials,
timing,
timing_entry.gpu_dispatch_capability.as_deref(),
timing_entry.gpu_slot_input_capacity_bytes,
timing_entry.gpu_slot_match_capacity,
);
Self {
backend: route.backend.label().to_string(),
phase2_plain_localizer: route.phase2_plain_localizer,
phase2_keyword_localizer: route.phase2_keyword_localizer,
peer_identity: peer_identity.map(str::to_owned),
gpu_pipeline_depth: route.gpu_pipeline_depth,
gpu_dispatch_capability: timing_entry.gpu_dispatch_capability.clone(),
gpu_slot_input_capacity_bytes: timing_entry.gpu_slot_input_capacity_bytes,
gpu_slot_match_capacity: timing_entry.gpu_slot_match_capacity,
correctness_digest,
completed_trials,
evidence_digest,
}
}
pub(super) fn expected_evidence_digest(
&self,
route: MeasuredRoute,
timing: &BackendTimingEvidence,
) -> u64 {
Self::evidence_digest_for(
route,
self.peer_identity.as_deref(),
self.correctness_digest,
self.completed_trials,
timing,
self.gpu_dispatch_capability.as_deref(),
self.gpu_slot_input_capacity_bytes,
self.gpu_slot_match_capacity,
)
}
fn evidence_digest_for(
route: MeasuredRoute,
peer_identity: Option<&str>,
correctness_digest: u64,
completed_trials: usize,
timing: &BackendTimingEvidence,
gpu_dispatch_capability: Option<&str>,
gpu_slot_input_capacity_bytes: Option<u64>,
gpu_slot_match_capacity: Option<u32>,
) -> u64 {
let mut hasher = crate::stable_hash::StableHasher::new("autoroute-parity-receipt");
hasher
.field_str("backend", route.backend.label())
.field_bool("phase2_plain_localizer", route.phase2_plain_localizer)
.field_bool("phase2_keyword_localizer", route.phase2_keyword_localizer)
.field_u64("gpu_pipeline_depth", u64::from(route.gpu_pipeline_depth))
.field_bool(
"gpu_dispatch_capability.present",
gpu_dispatch_capability.is_some(),
)
.field_str(
"gpu_dispatch_capability",
gpu_dispatch_capability.unwrap_or(""),
)
.field_u64(
"gpu_slot_input_capacity_bytes",
gpu_slot_input_capacity_bytes.unwrap_or(0),
)
.field_u64(
"gpu_slot_match_capacity",
u64::from(gpu_slot_match_capacity.unwrap_or(0)),
)
.field_bool("peer_identity.present", peer_identity.is_some())
.field_str("peer_identity", peer_identity.unwrap_or(""))
.field_u64("correctness_digest", correctness_digest)
.field_usize("completed_trials", completed_trials)
.field_usize("timing.trials_ns.len", timing.trials_ns.len());
for (index, trial_ns) in timing.trials_ns.iter().enumerate() {
hasher
.field_usize("timing.trial.index", index)
.field_bytes("timing.trial.ns", &trial_ns.to_le_bytes());
}
hasher.finish_u64()
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct AutorouteDecision {
pub(super) backend: String,
pub(super) phase2_plain_localizer: bool,
pub(super) phase2_keyword_localizer: bool,
pub(super) gpu_pipeline_depth: u8,
pub(super) calibration_points: Vec<AutorouteCalibrationPoint>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct RouteTimingEvidence {
pub(super) backend: String,
pub(super) phase2_plain_localizer: bool,
pub(super) phase2_keyword_localizer: bool,
pub(super) gpu_pipeline_depth: u8,
pub(super) gpu_dispatch_capability: Option<String>,
pub(super) gpu_slot_input_capacity_bytes: Option<u64>,
pub(super) gpu_slot_match_capacity: Option<u32>,
pub(super) peer_identity: Option<String>,
pub(super) ordered_device_route: Option<keyhog_scanner::gpu::device_set::OrderedGpuDeviceRoute>,
pub(super) timing: BackendTimingEvidence,
}
impl RouteTimingEvidence {
#[cfg(test)]
pub(super) fn new(route: MeasuredRoute, timing: BackendTimingEvidence) -> Self {
let peer_identity = route
.backend
.is_gpu()
.then(|| format!("test-peer:{}", route.backend.label()));
let gpu_pipeline = route.backend.is_gpu().then(|| {
(
if route.gpu_pipeline_depth == 1 {
"timed-resident"
} else {
"async-submit-retire"
}
.to_string(),
1024_u64 / u64::from(route.gpu_pipeline_depth),
65_536_u32 / u32::from(route.gpu_pipeline_depth),
)
});
Self::new_with_peer_identity(route, timing, peer_identity, gpu_pipeline)
}
pub(super) fn new_with_peer_identity(
route: MeasuredRoute,
timing: BackendTimingEvidence,
peer_identity: Option<String>,
gpu_pipeline: Option<(String, u64, u32)>,
) -> Self {
let (gpu_dispatch_capability, gpu_slot_input_capacity_bytes, gpu_slot_match_capacity) =
match gpu_pipeline {
Some((capability, input_capacity, match_capacity)) => {
(Some(capability), Some(input_capacity), Some(match_capacity))
}
None => (None, None, None),
};
Self {
backend: route.backend.label().to_string(),
phase2_plain_localizer: route.phase2_plain_localizer,
phase2_keyword_localizer: route.phase2_keyword_localizer,
gpu_pipeline_depth: route.gpu_pipeline_depth,
gpu_dispatch_capability,
gpu_slot_input_capacity_bytes,
gpu_slot_match_capacity,
peer_identity,
ordered_device_route: None,
timing,
}
}
#[allow(dead_code)]
pub(super) fn bind_ordered_device_route(
mut self,
device_route: keyhog_scanner::gpu::device_set::OrderedGpuDeviceRoute,
) -> Result<Self, String> {
device_route.validate()?;
let measured = self
.measured_route()
.ok_or_else(|| "ordered GPU route names an unsupported backend".to_string())?;
if !measured.backend.is_gpu() {
return Err("ordered GPU device evidence cannot bind a host backend".to_string());
}
if device_route.devices.len() < 2 {
return Err(
"ordered multi-device autoroute evidence requires at least two devices".to_string(),
);
}
if device_route
.devices
.iter()
.any(|device| device.api.scan_backend() != measured.backend)
{
return Err(format!(
"ordered device set does not use the measured {} backend on every device",
measured.backend.label()
));
}
self.peer_identity = Some(format!(
"ordered-device-set:{}",
device_route.authenticated_digest
));
self.ordered_device_route = Some(device_route);
Ok(self)
}
pub(super) fn measured_route(&self) -> Option<MeasuredRoute> {
Some(MeasuredRoute {
backend: keyhog_scanner::hw_probe::parse_backend_str(&self.backend)?,
phase2_plain_localizer: self.phase2_plain_localizer,
phase2_keyword_localizer: self.phase2_keyword_localizer,
gpu_pipeline_depth: self.gpu_pipeline_depth,
})
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct AutorouteCalibrationPoint {
pub(super) sample_bytes: u64,
pub(super) sample_chunks: usize,
pub(super) measurement_shape: MeasurementShapeEvidence,
pub(super) compiled_default_phase2_plain_localizer: bool,
pub(super) compiled_default_phase2_keyword_localizer: bool,
pub(super) candidate_receipts: Vec<BackendParityReceipt>,
pub(super) calibrated_at_unix_ms: u128,
pub(super) route_timings: Vec<RouteTimingEvidence>,
pub(super) trials: usize,
}
impl AutorouteCalibrationPoint {
fn measured_routes(&self) -> Vec<MeasuredRoute> {
self.route_timings
.iter()
.filter_map(RouteTimingEvidence::measured_route)
.collect()
}
pub(super) fn route_timing_for_route(
&self,
route: MeasuredRoute,
) -> Option<&RouteTimingEvidence> {
self.route_timings
.iter()
.find(|entry| entry.measured_route() == Some(route))
}
pub(super) fn timing_for_route(&self, route: MeasuredRoute) -> Option<&BackendTimingEvidence> {
self.route_timing_for_route(route)
.map(|entry| &entry.timing)
}
pub(super) fn baseline_timing_for_backend(
&self,
backend: ScanBackend,
) -> Option<&BackendTimingEvidence> {
self.timing_for_route(MeasuredRoute {
backend,
phase2_plain_localizer: false,
phase2_keyword_localizer: false,
gpu_pipeline_depth: 1,
})
}
pub(super) fn gpu_cold_warm_route_for_measured(
&self,
route: MeasuredRoute,
) -> Option<(u128, BackendTimingEvidence, u128)> {
route.backend.is_gpu().then_some(())?;
self.timing_for_route(route)
.and_then(gpu_cold_warm_route_evidence)
}
pub(super) fn accelerator_cold_warm_route_for_measured(
&self,
route: MeasuredRoute,
) -> Option<(u128, BackendTimingEvidence, u128)> {
match route.backend {
ScanBackend::SimdCpu => self
.timing_for_route(route)
.and_then(simd_cold_warm_route_evidence),
ScanBackend::GpuCuda | ScanBackend::GpuMetal | ScanBackend::GpuWgpu => {
self.gpu_cold_warm_route_for_measured(route)
}
_ => None,
}
}
pub(super) fn selected_route_has_confidence_for(
&self,
selected: MeasuredRoute,
persistent_runtime: bool,
) -> bool {
self.resolve_measured_route(persistent_runtime) == Some(selected)
}
pub(super) fn selected_route_has_exact_plan_confidence_for(
&self,
selected: MeasuredRoute,
persistent_runtime: bool,
) -> bool {
self.route_is_confidence_winner(selected, persistent_runtime, None)
}
pub(super) fn resolve_measured_route(&self, persistent_runtime: bool) -> Option<MeasuredRoute> {
self.resolve_measured_route_excluding(persistent_runtime, None)
}
pub(super) fn resolve_selected_route(&self, persistent_runtime: bool) -> Option<MeasuredRoute> {
self.resolve_selected_route_excluding(persistent_runtime, None)
}
pub(super) fn resolve_selected_route_excluding(
&self,
persistent_runtime: bool,
excluded_backend: Option<ScanBackend>,
) -> Option<MeasuredRoute> {
self.resolve_measured_route_excluding(persistent_runtime, excluded_backend)
.or_else(|| self.resolve_dead_heat_route(persistent_runtime, excluded_backend))
}
fn resolve_measured_route_excluding(
&self,
persistent_runtime: bool,
excluded_backend: Option<ScanBackend>,
) -> Option<MeasuredRoute> {
let candidates = self.route_candidates_for_runtime(persistent_runtime);
candidates
.iter()
.copied()
.filter(|(route, _)| Some(route.backend) != excluded_backend)
.filter(|(route, _)| {
self.route_is_confidence_winner(*route, persistent_runtime, excluded_backend)
})
.min_by_key(|(route, median_ns)| {
(
*median_ns,
route.phase2_plain_localizer,
route.phase2_keyword_localizer,
route.gpu_pipeline_depth,
)
})
.map(|(route, _)| route)
.or_else(|| {
self.resolve_peer_separated_tied_route(persistent_runtime, excluded_backend)
})
}
fn resolve_peer_separated_tied_route(
&self,
persistent_runtime: bool,
excluded_backend: Option<ScanBackend>,
) -> Option<MeasuredRoute> {
let intervals = self
.route_confidence_intervals_for(persistent_runtime)
.into_iter()
.filter(|(route, _)| Some(route.backend) != excluded_backend)
.collect::<Vec<_>>();
intervals
.iter()
.filter(|(selected, selected_interval)| {
let has_peer = intervals
.iter()
.any(|(route, _)| route.backend != selected.backend);
(has_peer || excluded_backend.is_some())
&& intervals
.iter()
.filter(|(route, _)| route.backend != selected.backend)
.all(|(_, competitor_interval)| {
selected_interval.high_ns < competitor_interval.low_ns
})
&& intervals
.iter()
.filter(|(route, _)| {
route.backend == selected.backend && *route != *selected
})
.all(|(competitor, _)| {
!self.same_backend_plan_is_faster(
*competitor,
*selected,
persistent_runtime,
)
})
})
.min_by_key(|(route, _)| {
(
route.phase2_plain_localizer != self.compiled_default_phase2_plain_localizer
|| route.phase2_keyword_localizer
!= self.compiled_default_phase2_keyword_localizer,
route.phase2_plain_localizer,
route.phase2_keyword_localizer,
route.gpu_pipeline_depth,
)
})
.map(|(route, _)| *route)
}
fn resolve_dead_heat_route(
&self,
persistent_runtime: bool,
excluded_backend: Option<ScanBackend>,
) -> Option<MeasuredRoute> {
let intervals = self
.route_confidence_intervals_for(persistent_runtime)
.into_iter()
.filter(|(route, _)| Some(route.backend) != excluded_backend)
.collect::<Vec<_>>();
let contenders = intervals
.iter()
.filter_map(|(route, interval)| {
self.route_median_ns(*route, persistent_runtime)
.map(|median_ns| (*route, median_ns, *interval))
})
.filter(|(_, _, interval)| {
!intervals
.iter()
.any(|(_, peer_interval)| peer_interval.high_ns < interval.low_ns)
})
.collect::<Vec<_>>();
let fastest_high_ns = contenders
.iter()
.min_by_key(|(route, median_ns, _)| {
(
*median_ns,
backend_route_complexity(route.backend),
route.phase2_plain_localizer,
route.phase2_keyword_localizer,
route.gpu_pipeline_depth,
)
})
.map(|(_, _, interval)| interval.high_ns)?;
contenders
.iter()
.filter(|(_, median_ns, _)| *median_ns <= fastest_high_ns)
.min_by_key(|(route, median_ns, _)| {
(
backend_route_complexity(route.backend),
route.phase2_plain_localizer != self.compiled_default_phase2_plain_localizer
|| route.phase2_keyword_localizer
!= self.compiled_default_phase2_keyword_localizer,
*median_ns,
route.phase2_plain_localizer,
route.phase2_keyword_localizer,
route.gpu_pipeline_depth,
)
})
.map(|(route, _, _)| *route)
}
fn route_trial_ns_for(
&self,
route: MeasuredRoute,
persistent_runtime: bool,
) -> Option<Vec<u128>> {
if route.backend == ScanBackend::SimdCpu || route.backend.is_gpu() {
let (cold_ns, warm_timing, _) = self.accelerator_cold_warm_route_for_measured(route)?;
Some(
warm_timing
.trials_ns
.into_iter()
.map(|warm_ns| {
if persistent_runtime {
warm_ns
} else {
cold_ns.max(warm_ns)
}
})
.collect(),
)
} else {
self.timing_for_route(route)
.map(|timing| timing.trials_ns.clone())
}
}
fn route_is_confidence_winner(
&self,
selected: MeasuredRoute,
persistent_runtime: bool,
excluded_backend: Option<ScanBackend>,
) -> bool {
let intervals = self
.route_confidence_intervals_for(persistent_runtime)
.into_iter()
.filter(|(route, _)| Some(route.backend) != excluded_backend)
.collect::<Vec<_>>();
let Some((_, selected_interval)) = intervals
.iter()
.find(|(route, _)| *route == selected)
.copied()
else {
return false;
};
intervals
.iter()
.filter(|(route, _)| *route != selected)
.all(|(competitor, competitor_interval)| {
if competitor.backend != selected.backend {
return selected_interval.high_ns < competitor_interval.low_ns;
}
self.same_backend_plan_is_faster(selected, *competitor, persistent_runtime)
})
}
fn route_median_ns(&self, route: MeasuredRoute, persistent_runtime: bool) -> Option<u128> {
match route.backend {
ScanBackend::CpuFallback => self
.timing_for_route(route)
.map(BackendTimingEvidence::median_ns),
ScanBackend::SimdCpu
| ScanBackend::GpuCuda
| ScanBackend::GpuMetal
| ScanBackend::GpuWgpu => {
let (_, warm_timing, one_shot_ns) =
self.accelerator_cold_warm_route_for_measured(route)?;
Some(if persistent_runtime {
warm_timing.median_ns()
} else {
one_shot_ns
})
}
_ => None,
}
}
fn same_backend_plan_is_faster(
&self,
faster: MeasuredRoute,
slower: MeasuredRoute,
persistent_runtime: bool,
) -> bool {
let intervals = self.route_confidence_intervals_for(persistent_runtime);
let interval_for = |route: MeasuredRoute| {
intervals
.iter()
.find(|(candidate, _)| *candidate == route)
.map(|(_, interval)| *interval)
};
let (Some(faster_interval), Some(slower_interval)) =
(interval_for(faster), interval_for(slower))
else {
return false;
};
if faster_interval.high_ns >= slower_interval.low_ns {
return false;
}
let (Some(faster_trials), Some(slower_trials)) = (
self.route_trial_ns_for(faster, persistent_runtime),
self.route_trial_ns_for(slower, persistent_runtime),
) else {
return false;
};
paired_route_trials_are_faster(&faster_trials, &slower_trials)
}
fn route_confidence_intervals_for(
&self,
persistent_runtime: bool,
) -> Vec<(MeasuredRoute, TimingConfidenceInterval)> {
let mut intervals = Vec::with_capacity(self.route_timings.len());
for route in self.measured_routes() {
if route.backend == ScanBackend::SimdCpu || route.backend.is_gpu() {
let Some((cold_ns, warm_timing, _route_ns)) =
self.accelerator_cold_warm_route_for_measured(route)
else {
continue;
};
let warm_interval = warm_timing.confidence_interval_95_ns();
intervals.push((
route,
if persistent_runtime {
warm_interval
} else {
TimingConfidenceInterval {
low_ns: cold_ns.max(warm_interval.low_ns),
high_ns: cold_ns.max(warm_interval.high_ns),
}
},
));
} else if let Some(timing) = self.timing_for_route(route) {
intervals.push((route, timing.confidence_interval_95_ns()));
}
}
intervals
}
fn route_candidates_for_runtime(&self, persistent_runtime: bool) -> Vec<(MeasuredRoute, u128)> {
self.measured_routes()
.into_iter()
.filter_map(|route| {
self.route_median_ns(route, persistent_runtime)
.map(|timing| (route, timing))
})
.collect()
}
}
impl AutorouteDecision {
fn candidate_receipts(
correctness_digest: u64,
route_timings: &[RouteTimingEvidence],
) -> Vec<BackendParityReceipt> {
route_timings
.iter()
.filter_map(|entry| {
Some(BackendParityReceipt::new(
entry.measured_route()?,
entry,
correctness_digest,
))
})
.collect()
}
fn canonicalize_route_timings(route_timings: &mut [RouteTimingEvidence]) {
route_timings.sort_unstable_by(|left, right| {
(
left.backend.as_str(),
left.phase2_plain_localizer,
left.phase2_keyword_localizer,
left.gpu_pipeline_depth,
)
.cmp(&(
right.backend.as_str(),
right.phase2_plain_localizer,
right.phase2_keyword_localizer,
right.gpu_pipeline_depth,
))
});
}
#[cfg(test)]
fn test_route_timings(
backends: impl IntoIterator<Item = (ScanBackend, Option<BackendTimingEvidence>)>,
) -> Vec<RouteTimingEvidence> {
let mut routes = Vec::new();
for (backend, timing) in backends {
let Some(base) = timing else {
continue;
};
for phase2_plain_localizer in [false, true] {
for phase2_keyword_localizer in [false, true] {
let timing = if phase2_plain_localizer || phase2_keyword_localizer {
BackendTimingEvidence::constant_ms(
base.median_ms().saturating_add(1_000),
AUTOROUTE_CALIBRATION_TRIALS,
)
} else {
base.clone()
};
routes.push(RouteTimingEvidence::new(
MeasuredRoute {
backend,
phase2_plain_localizer,
phase2_keyword_localizer,
gpu_pipeline_depth: 1,
},
timing,
));
}
}
}
routes
}
#[cfg(test)]
pub(super) fn new(
backend: ScanBackend,
sample_bytes: u64,
sample_chunks: usize,
simd_ms: u128,
cpu_ms: Option<u128>,
gpu_ms: Option<u128>,
) -> Self {
let simd_timing = BackendTimingEvidence::constant_ms(simd_ms, AUTOROUTE_CALIBRATION_TRIALS);
let cpu_duration_ms = match cpu_ms {
Some(duration_ms) => duration_ms,
None => simd_ms.saturating_add(1_000),
};
let cpu_timing = Some(BackendTimingEvidence::constant_ms(
cpu_duration_ms,
AUTOROUTE_CALIBRATION_TRIALS,
));
let gpu_wgpu_timing =
gpu_ms.map(|ms| BackendTimingEvidence::constant_ms(ms, AUTOROUTE_CALIBRATION_TRIALS));
let mut route_timings = Self::test_route_timings([
(ScanBackend::SimdCpu, Some(simd_timing)),
(ScanBackend::CpuFallback, cpu_timing),
(ScanBackend::GpuCuda, None),
(ScanBackend::GpuWgpu, gpu_wgpu_timing),
]);
Self::canonicalize_route_timings(&mut route_timings);
let candidate_receipts = Self::candidate_receipts(0xA11D_0B57_A11D_0B57, &route_timings);
Self {
backend: backend.label().to_string(),
phase2_plain_localizer: false,
phase2_keyword_localizer: false,
gpu_pipeline_depth: 1,
calibration_points: vec![AutorouteCalibrationPoint {
sample_bytes,
sample_chunks,
measurement_shape: super::workload::test_measurement_shape_evidence(
sample_bytes,
sample_chunks,
),
compiled_default_phase2_plain_localizer: false,
compiled_default_phase2_keyword_localizer: false,
candidate_receipts,
calibrated_at_unix_ms: 1,
route_timings,
trials: AUTOROUTE_CALIBRATION_TRIALS,
}],
}
}
#[cfg(test)]
pub(super) fn from_timing_evidence(
backend: ScanBackend,
sample_bytes: u64,
sample_chunks: usize,
correctness_digest: u64,
calibrated_at_unix_ms: u128,
simd_timing: BackendTimingEvidence,
cpu_timing: Option<BackendTimingEvidence>,
gpu_timing: Option<BackendTimingEvidence>,
) -> Self {
let mut route_timings = Self::test_route_timings([
(ScanBackend::SimdCpu, Some(simd_timing)),
(ScanBackend::CpuFallback, cpu_timing),
(ScanBackend::GpuCuda, None),
(ScanBackend::GpuWgpu, gpu_timing),
]);
Self::canonicalize_route_timings(&mut route_timings);
let candidate_receipts = Self::candidate_receipts(correctness_digest, &route_timings);
Self {
backend: backend.label().to_string(),
phase2_plain_localizer: false,
phase2_keyword_localizer: false,
gpu_pipeline_depth: 1,
calibration_points: vec![AutorouteCalibrationPoint {
sample_bytes,
sample_chunks,
measurement_shape: super::workload::test_measurement_shape_evidence(
sample_bytes,
sample_chunks,
),
compiled_default_phase2_plain_localizer: false,
compiled_default_phase2_keyword_localizer: false,
candidate_receipts,
calibrated_at_unix_ms,
route_timings,
trials: AUTOROUTE_CALIBRATION_TRIALS,
}],
}
}
pub(super) fn from_peer_timing_evidence(
backend: ScanBackend,
sample_bytes: u64,
sample_chunks: usize,
measurement_shape: MeasurementShapeEvidence,
correctness_digest: u64,
calibrated_at_unix_ms: u128,
mut route_timings: Vec<RouteTimingEvidence>,
compiled_default_phase2_plain_localizer: bool,
compiled_default_phase2_keyword_localizer: bool,
) -> Self {
Self::canonicalize_route_timings(&mut route_timings);
let candidate_receipts = Self::candidate_receipts(correctness_digest, &route_timings);
Self {
backend: backend.label().to_string(),
phase2_plain_localizer: false,
phase2_keyword_localizer: false,
gpu_pipeline_depth: 1,
calibration_points: vec![AutorouteCalibrationPoint {
sample_bytes,
sample_chunks,
measurement_shape,
compiled_default_phase2_plain_localizer,
compiled_default_phase2_keyword_localizer,
candidate_receipts,
calibrated_at_unix_ms,
route_timings,
trials: AUTOROUTE_CALIBRATION_TRIALS,
}],
}
}
pub(super) fn contains_measurement(
&self,
measurement_shape: &MeasurementShapeEvidence,
) -> bool {
self.calibration_points
.iter()
.any(|point| point.measurement_shape.shape_digest == measurement_shape.shape_digest)
}
pub(super) fn merge_calibration_point(
&mut self,
point: AutorouteDecision,
) -> Result<(), String> {
if point.calibration_points.len() != 1 {
return Err("cannot merge a nested autoroute calibration envelope".into());
}
let declared_one_shot = point
.measured_route()
.ok_or_else(|| "new workload point declares an unsupported route".to_string())?;
let point = point
.calibration_points
.into_iter()
.next()
.ok_or_else(|| "autoroute calibration envelope lost its only point".to_string())?;
if self.contains_measurement(&point.measurement_shape) {
return Ok(());
}
for incoming in &point.route_timings {
let route = incoming
.measured_route()
.ok_or_else(|| "new workload point contains an unsupported route".to_string())?;
let existing = self
.calibration_points
.first()
.and_then(|point| point.route_timing_for_route(route))
.ok_or_else(|| {
format!(
"new workload point contains route {} absent from existing evidence",
render_measured_route(route)
)
})?;
match (
existing.ordered_device_route.as_ref(),
incoming.ordered_device_route.as_ref(),
) {
(None, None) => {}
(Some(left), Some(right)) if left.has_same_device_set_identity(right) => {}
(Some(_), Some(_)) => {
return Err(format!(
"workload class changes its ordered GPU device-set identity for {}",
render_measured_route(route)
));
}
_ => {
return Err(format!(
"workload class changes between single-device and ordered multi-device evidence for {}",
render_measured_route(route)
));
}
}
}
if self.calibration_points.len() >= MAX_AUTOROUTE_MEASURED_POINTS {
return Err(format!(
"autoroute workload class already contains the maximum {MAX_AUTOROUTE_MEASURED_POINTS} measured calibration points; split the workload identity before adding more evidence"
));
}
let expected_one_shot = self.resolved_routing_route().ok_or_else(|| {
"existing workload evidence does not resolve one one-shot route across its measured points"
.to_string()
})?;
let measured_one_shot = point
.resolve_selected_route(false)
.ok_or_else(|| "new workload point does not resolve one one-shot route".to_string())?;
if declared_one_shot != measured_one_shot {
return Err(format!(
"new workload point declares {} but its timing evidence resolves {}; recalibrate the point",
render_measured_route(declared_one_shot),
render_measured_route(measured_one_shot)
));
}
let expected_daemon = self.resolved_persistent_route().ok_or_else(|| {
"existing workload evidence does not resolve one daemon route across its measured points"
.to_string()
})?;
let measured_daemon = point
.resolve_selected_route(true)
.ok_or_else(|| "new workload point does not resolve one daemon route".to_string())?;
if expected_one_shot.backend != measured_one_shot.backend
|| expected_daemon.backend != measured_daemon.backend
{
return Err(format!(
"workload class changes its confidence-supported backend across measured points: existing one-shot={} daemon={}, new {}-byte/{}-chunk point one-shot={} daemon={}; split the workload identity at this crossover and recalibrate",
render_measured_route(expected_one_shot),
render_measured_route(expected_daemon),
point.sample_bytes,
point.sample_chunks,
render_measured_route(measured_one_shot),
render_measured_route(measured_daemon),
));
}
for (runtime_label, persistent_runtime, expected_route) in [
("one-shot", false, expected_one_shot),
("daemon", true, expected_daemon),
] {
if expected_route.backend == ScanBackend::CpuFallback {
continue;
}
let existing_recovery = self
.resolved_recovery_route(expected_route.backend, persistent_runtime)
.ok_or_else(|| {
format!(
"existing workload evidence has no unanimous {runtime_label} recovery route after {}",
expected_route.backend.label()
)
})?;
let measured_recovery = point
.resolve_selected_route_excluding(persistent_runtime, Some(expected_route.backend))
.ok_or_else(|| {
format!(
"new workload point has no {runtime_label} recovery route after {}",
expected_route.backend.label()
)
})?;
if existing_recovery.backend != measured_recovery.backend {
return Err(format!(
"workload class changes its confidence-supported remaining {runtime_label} recovery backend after {}: existing={}, new {}-byte/{}-chunk point={}; split the workload identity at this recovery crossover and recalibrate",
expected_route.backend.label(),
render_measured_route(existing_recovery),
point.sample_bytes,
point.sample_chunks,
render_measured_route(measured_recovery),
));
}
}
self.calibration_points.push(point);
self.calibration_points.sort_unstable_by_key(|point| {
(
point.sample_bytes,
point.sample_chunks,
point.measurement_shape.shape_digest,
)
});
let reconciled = self.resolved_routing_route().ok_or_else(|| {
"merged workload evidence does not resolve one one-shot route".to_string()
})?;
self.backend = reconciled.backend.label().to_string();
self.phase2_plain_localizer = reconciled.phase2_plain_localizer;
self.phase2_keyword_localizer = reconciled.phase2_keyword_localizer;
self.gpu_pipeline_depth = reconciled.gpu_pipeline_depth;
Ok(())
}
pub(super) fn backend(&self) -> Option<ScanBackend> {
keyhog_scanner::hw_probe::parse_backend_str(&self.backend)
}
pub(super) fn measured_route(&self) -> Option<MeasuredRoute> {
Some(MeasuredRoute {
backend: self.backend()?,
phase2_plain_localizer: self.phase2_plain_localizer,
phase2_keyword_localizer: self.phase2_keyword_localizer,
gpu_pipeline_depth: self.gpu_pipeline_depth,
})
}
#[allow(dead_code)]
pub(super) fn peer_identity_for_route(&self, route: MeasuredRoute) -> Option<&str> {
let first = self
.calibration_points
.first()?
.route_timings
.iter()
.find(|entry| entry.measured_route() == Some(route))?
.peer_identity
.as_deref();
self.calibration_points
.iter()
.all(|point| {
point
.route_timings
.iter()
.find(|entry| entry.measured_route() == Some(route))
.and_then(|entry| entry.peer_identity.as_deref())
== first
})
.then_some(first)
.flatten()
}
#[allow(dead_code)]
pub(super) fn ordered_device_route_for_route(
&self,
route: MeasuredRoute,
) -> Option<&keyhog_scanner::gpu::device_set::OrderedGpuDeviceRoute> {
let first = self
.calibration_points
.first()?
.route_timing_for_route(route)?
.ordered_device_route
.as_ref()?;
self.calibration_points
.iter()
.all(|point| {
point
.route_timing_for_route(route)
.and_then(|entry| entry.ordered_device_route.as_ref())
.is_some_and(|candidate| first.has_same_device_set_identity(candidate))
})
.then_some(first)
}
pub(super) fn gpu_pipeline_identity_for_route(
&self,
route: MeasuredRoute,
) -> Option<(&str, u64, u32)> {
let first = self
.calibration_points
.first()?
.route_timings
.iter()
.find(|entry| entry.measured_route() == Some(route))?;
let identity = (
first.gpu_dispatch_capability.as_deref()?,
first.gpu_slot_input_capacity_bytes?,
first.gpu_slot_match_capacity?,
);
self.calibration_points
.iter()
.all(|point| {
point
.route_timings
.iter()
.find(|entry| entry.measured_route() == Some(route))
.is_some_and(|entry| {
entry.gpu_dispatch_capability.as_deref() == Some(identity.0)
&& entry.gpu_slot_input_capacity_bytes == Some(identity.1)
&& entry.gpu_slot_match_capacity == Some(identity.2)
})
})
.then_some(identity)
}
pub(super) fn primary_point(&self) -> &AutorouteCalibrationPoint {
self.calibration_points.first().unwrap_or_else(|| {
panic!("autoroute decisions are constructed and validated with evidence")
})
}
#[cfg(test)]
pub(super) fn primary_point_mut(&mut self) -> &mut AutorouteCalibrationPoint {
self.calibration_points
.first_mut()
.unwrap_or_else(|| panic!("test autoroute decision must contain evidence"))
}
pub(super) fn simd_baseline_ms(&self) -> u128 {
self.primary_point()
.baseline_timing_for_backend(ScanBackend::SimdCpu)
.unwrap_or_else(|| panic!("validated calibration contains the SIMD baseline route"))
.median_ms()
}
pub(super) fn cpu_baseline_ms(&self) -> Option<u128> {
self.primary_point()
.baseline_timing_for_backend(ScanBackend::CpuFallback)
.map(BackendTimingEvidence::median_ms)
}
#[cfg(test)]
pub(super) fn gpu_ms(&self) -> Option<u128> {
self.gpu_route_ns().map(|route_ns| route_ns / 1_000_000)
}
#[cfg(test)]
pub(super) fn gpu_cold_warm_route(&self) -> Option<(u128, BackendTimingEvidence, u128)> {
let route = self.measured_route()?;
route.backend.is_gpu().then_some(())?;
self.primary_point()
.timing_for_route(route)
.and_then(gpu_cold_warm_route_evidence)
}
#[cfg(test)]
pub(super) fn gpu_cold_ns(&self) -> Option<u128> {
self.gpu_cold_warm_route().map(|(cold_ns, _, _)| cold_ns)
}
#[cfg(test)]
pub(super) fn gpu_warm_ms(&self) -> Option<u128> {
self.gpu_cold_warm_route()
.map(|(_, warm_timing, _)| warm_timing.median_ms())
}
#[cfg(test)]
pub(super) fn gpu_route_ns(&self) -> Option<u128> {
self.gpu_cold_warm_route().map(|(_, _, route_ns)| route_ns)
}
pub(super) fn selected_margin_ns(&self) -> Option<u128> {
let route = self.measured_route()?;
self.calibration_points
.iter()
.map(|point| {
selected_route_margin_ns(route, &point.route_candidates_for_runtime(false))
})
.collect::<Option<Vec<_>>>()?
.into_iter()
.min()
}
pub(super) fn persistent_selected_margin_ns(&self) -> Option<u128> {
let route = self.resolved_persistent_route()?;
self.calibration_points
.iter()
.map(|point| selected_route_margin_ns(route, &point.route_candidates_for_runtime(true)))
.collect::<Option<Vec<_>>>()?
.into_iter()
.min()
}
pub(super) fn baseline_timing_for_backend(
&self,
backend: ScanBackend,
) -> Option<&BackendTimingEvidence> {
self.primary_point().baseline_timing_for_backend(backend)
}
#[cfg(test)]
pub(super) fn selected_backend_has_non_overlapping_confidence(
&self,
selected: ScanBackend,
) -> bool {
let Some(route) = self
.measured_route()
.filter(|route| route.backend == selected)
else {
return false;
};
self.selected_route_has_confidence_for(route, false)
}
fn selected_route_has_confidence_for(
&self,
selected: MeasuredRoute,
persistent_runtime: bool,
) -> bool {
self.calibration_points
.iter()
.all(|point| point.selected_route_has_confidence_for(selected, persistent_runtime))
}
pub(super) fn resolved_routing_route(&self) -> Option<MeasuredRoute> {
self.resolve_class_route(false)
}
fn resolve_class_route(&self, persistent_runtime: bool) -> Option<MeasuredRoute> {
let resolve =
|point: &AutorouteCalibrationPoint| point.resolve_selected_route(persistent_runtime);
let first = self.calibration_points.first()?;
let selected = resolve(first)?;
let resolved: Vec<MeasuredRoute> = self
.calibration_points
.iter()
.map(resolve)
.collect::<Option<Vec<_>>>()?;
if resolved
.iter()
.any(|route| route.backend != selected.backend)
{
return None;
}
if resolved.iter().all(|route| *route == selected) {
return Some(selected);
}
let default_plan = MeasuredRoute {
backend: selected.backend,
phase2_plain_localizer: first.compiled_default_phase2_plain_localizer,
phase2_keyword_localizer: first.compiled_default_phase2_keyword_localizer,
gpu_pipeline_depth: selected.gpu_pipeline_depth,
};
self.calibration_points
.iter()
.all(|point| {
point.compiled_default_phase2_plain_localizer == default_plan.phase2_plain_localizer
&& point.compiled_default_phase2_keyword_localizer
== default_plan.phase2_keyword_localizer
&& point.measured_routes().contains(&default_plan)
})
.then_some(default_plan)
}
#[cfg(test)]
pub(super) fn resolved_routing_backend(&self) -> Option<ScanBackend> {
self.resolved_routing_route().map(|route| route.backend)
}
pub(super) fn resolved_persistent_route(&self) -> Option<MeasuredRoute> {
self.resolve_class_route(true)
}
pub(super) fn resolved_persistent_backend(&self) -> Option<ScanBackend> {
self.resolved_persistent_route().map(|route| route.backend)
}
pub(super) fn resolved_recovery_route(
&self,
failed_backend: ScanBackend,
persistent_runtime: bool,
) -> Option<MeasuredRoute> {
let selected = self
.calibration_points
.first()?
.resolve_selected_route_excluding(persistent_runtime, Some(failed_backend))?;
self.calibration_points
.iter()
.all(|point| {
point.resolve_selected_route_excluding(persistent_runtime, Some(failed_backend))
== Some(selected)
})
.then_some(selected)
}
pub(super) fn has_confidence_supported_route(&self) -> bool {
self.resolved_routing_route()
.is_some_and(|winner| self.selected_route_has_confidence_for(winner, false))
}
pub(super) fn has_confidence_supported_persistent_route(&self) -> bool {
self.resolved_persistent_route()
.is_some_and(|winner| self.selected_route_has_confidence_for(winner, true))
}
pub(super) fn confidence_diagnostic(&self, persistent_runtime: bool) -> String {
let Some(point) = self.calibration_points.first() else {
return "no measured calibration point".to_string();
};
point
.route_confidence_intervals_for(persistent_runtime)
.into_iter()
.filter_map(|(route, interval)| {
point
.route_median_ns(route, persistent_runtime)
.map(|median_ns| {
format!(
"{} median_ns={median_ns} ci95_ns=[{},{}]",
render_measured_route(route),
interval.low_ns,
interval.high_ns,
)
})
})
.collect::<Vec<_>>()
.join("; ")
}
}
fn render_measured_route(route: MeasuredRoute) -> String {
format!(
"{}+phase2-plain-localizer={}+phase2-keyword-localizer={}+gpu-pipeline-depth={}",
route.backend.label(),
route.phase2_plain_localizer,
route.phase2_keyword_localizer,
route.gpu_pipeline_depth,
)
}