#![allow(dead_code)]
use std::time::{Duration, Instant};
use tracing::warn;
use super::{
CLASSIFICATION_TIER_LATENCY, CLASSIFICATION_TIER_TOTAL, DECISION_BUDGET_VIOLATIONS,
DECISION_LATENCY, DECISION_PANIC_THRESHOLD_VIOLATIONS,
};
pub const NON_COMPILATION_BUDGET_MS: f64 = 1.0;
pub const COMPILATION_BUDGET_MS: f64 = 5.0;
pub const WORKER_SELECTION_BUDGET_MS: f64 = 10.0;
pub const NON_COMPILATION_PANIC_MS: f64 = 5.0;
pub const COMPILATION_PANIC_MS: f64 = 10.0;
pub const WORKER_SELECTION_PANIC_MS: f64 = 50.0;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DecisionType {
NonCompilation,
Compilation,
WorkerSelection,
}
impl DecisionType {
pub fn budget_ms(self) -> f64 {
match self {
DecisionType::NonCompilation => NON_COMPILATION_BUDGET_MS,
DecisionType::Compilation => COMPILATION_BUDGET_MS,
DecisionType::WorkerSelection => WORKER_SELECTION_BUDGET_MS,
}
}
pub fn panic_threshold_ms(self) -> f64 {
match self {
DecisionType::NonCompilation => NON_COMPILATION_PANIC_MS,
DecisionType::Compilation => COMPILATION_PANIC_MS,
DecisionType::WorkerSelection => WORKER_SELECTION_PANIC_MS,
}
}
fn index(self) -> usize {
match self {
DecisionType::NonCompilation => 0,
DecisionType::Compilation => 1,
DecisionType::WorkerSelection => 2,
}
}
pub fn label(self) -> &'static str {
match self {
DecisionType::NonCompilation => "non_compilation",
DecisionType::Compilation => "compilation",
DecisionType::WorkerSelection => "worker_selection",
}
}
}
const VIOLATION_LOG_INTERVAL_MS: u64 = 60_000;
static VIOLATION_LAST_LOG_MS: [std::sync::atomic::AtomicU64; 7] =
[const { std::sync::atomic::AtomicU64::new(0) }; 7];
static VIOLATION_SUPPRESSED: [std::sync::atomic::AtomicU64; 7] =
[const { std::sync::atomic::AtomicU64::new(0) }; 7];
fn violation_log_gate(slot: usize, now_ms: u64) -> Option<u64> {
use std::sync::atomic::Ordering::Relaxed;
let last = VIOLATION_LAST_LOG_MS[slot].load(Relaxed);
if last != 0 && now_ms.saturating_sub(last) < VIOLATION_LOG_INTERVAL_MS
|| VIOLATION_LAST_LOG_MS[slot]
.compare_exchange(last, now_ms.max(1), Relaxed, Relaxed)
.is_err()
{
VIOLATION_SUPPRESSED[slot].fetch_add(1, Relaxed);
return None;
}
Some(VIOLATION_SUPPRESSED[slot].swap(0, Relaxed))
}
fn unix_now_ms() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |elapsed| elapsed.as_millis() as u64)
}
pub fn record_decision_latency(decision_type: DecisionType, start: Instant) -> Duration {
let duration = start.elapsed();
let duration_secs = duration.as_secs_f64();
let duration_ms = duration_secs * 1000.0;
DECISION_LATENCY
.with_label_values(&[decision_type.label()])
.observe(duration_secs);
let budget_ms = decision_type.budget_ms();
if duration_ms > budget_ms {
DECISION_BUDGET_VIOLATIONS
.with_label_values(&[decision_type.label()])
.inc();
if let Some(suppressed) = violation_log_gate(decision_type.index() * 2, unix_now_ms()) {
warn!(
decision_type = decision_type.label(),
duration_ms = format!("{:.3}", duration_ms),
budget_ms = budget_ms,
suppressed,
"Decision latency budget violation"
);
}
}
let panic_ms = decision_type.panic_threshold_ms();
if duration_ms > panic_ms {
DECISION_PANIC_THRESHOLD_VIOLATIONS
.with_label_values(&[decision_type.label()])
.inc();
if let Some(suppressed) = violation_log_gate(decision_type.index() * 2 + 1, unix_now_ms()) {
tracing::error!(
decision_type = decision_type.label(),
duration_ms = format!("{:.3}", duration_ms),
panic_threshold_ms = panic_ms,
suppressed,
"CRITICAL: Decision latency exceeded panic threshold"
);
}
}
duration
}
pub fn record_classification_tier(tier: u8, duration: Duration) {
let tier_str = tier.to_string();
CLASSIFICATION_TIER_TOTAL
.with_label_values(&[&tier_str])
.inc();
CLASSIFICATION_TIER_LATENCY
.with_label_values(&[&tier_str])
.observe(duration.as_secs_f64());
}
pub struct DecisionTimer {
decision_type: DecisionType,
start: Option<Instant>,
}
impl DecisionTimer {
pub fn new(decision_type: DecisionType) -> Self {
Self {
decision_type,
start: Some(Instant::now()),
}
}
pub fn elapsed(&self) -> Duration {
self.start.map(|start| start.elapsed()).unwrap_or_default()
}
pub fn stop(mut self) -> Duration {
self.start
.take()
.map(|start| record_decision_latency(self.decision_type, start))
.unwrap_or_default()
}
}
impl Drop for DecisionTimer {
fn drop(&mut self) {
if let Some(start) = self.start.take() {
let _ = record_decision_latency(self.decision_type, start);
}
}
}
pub struct TierTimer {
tier: u8,
start: Instant,
}
impl TierTimer {
pub fn new(tier: u8) -> Self {
Self {
tier,
start: Instant::now(),
}
}
pub fn stop(self) {
record_classification_tier(self.tier, self.start.elapsed());
}
}
#[cfg(test)]
mod tests {
use super::*;
use rch_common::test_guard;
use std::thread;
#[test]
fn test_decision_type_budget() {
let _guard = test_guard!();
assert_eq!(DecisionType::NonCompilation.budget_ms(), 1.0);
assert_eq!(DecisionType::Compilation.budget_ms(), 5.0);
assert_eq!(DecisionType::WorkerSelection.budget_ms(), 10.0);
}
#[test]
fn test_decision_type_label() {
let _guard = test_guard!();
assert_eq!(DecisionType::NonCompilation.label(), "non_compilation");
assert_eq!(DecisionType::Compilation.label(), "compilation");
assert_eq!(DecisionType::WorkerSelection.label(), "worker_selection");
}
#[test]
fn test_record_decision_latency() {
let _guard = test_guard!();
let start = Instant::now();
thread::sleep(Duration::from_micros(100));
let duration = record_decision_latency(DecisionType::NonCompilation, start);
assert!(duration.as_micros() >= 100);
}
#[test]
fn test_decision_timer() {
let _guard = test_guard!();
let timer = DecisionTimer::new(DecisionType::Compilation);
thread::sleep(Duration::from_micros(50));
let elapsed = timer.elapsed();
assert!(elapsed.as_micros() >= 50);
}
#[test]
fn test_decision_timer_stop_records_once() {
let _guard = test_guard!();
let histogram = DECISION_LATENCY.with_label_values(&["worker_selection"]);
let before = histogram.get_sample_count();
let timer = DecisionTimer::new(DecisionType::WorkerSelection);
let _elapsed = timer.stop();
assert_eq!(histogram.get_sample_count(), before + 1);
}
#[test]
fn test_tier_timer() {
let _guard = test_guard!();
let timer = TierTimer::new(2);
thread::sleep(Duration::from_micros(10));
timer.stop();
}
#[test]
fn violation_logs_are_gated_per_slot_and_report_suppressions() {
let base = 1_000_000_000;
assert_eq!(violation_log_gate(6, base), Some(0));
assert_eq!(violation_log_gate(6, base + 1), None);
assert_eq!(
violation_log_gate(6, base + VIOLATION_LOG_INTERVAL_MS - 1),
None
);
assert_eq!(
violation_log_gate(6, base + VIOLATION_LOG_INTERVAL_MS),
Some(2)
);
assert_eq!(
violation_log_gate(6, base + VIOLATION_LOG_INTERVAL_MS + 5),
None
);
}
#[test]
fn test_panic_thresholds() {
let _guard = test_guard!();
assert!(
DecisionType::NonCompilation.panic_threshold_ms()
> DecisionType::NonCompilation.budget_ms()
);
assert!(
DecisionType::Compilation.panic_threshold_ms() > DecisionType::Compilation.budget_ms()
);
assert!(
DecisionType::WorkerSelection.panic_threshold_ms()
> DecisionType::WorkerSelection.budget_ms()
);
}
}