use super::*;
pub struct CpuTimeAccounting {
scheduler_tick_cpu_time: Arc<scheduler::runtime::service::SchedulerTickCpuTime>,
published_user_ns: AtomicU64,
published_system_ns: AtomicU64,
published_runtime_ns: AtomicU64,
adjusted: RawSpinLock<CpuTimeHighWater>,
}
impl CpuTimeAccounting {
#[cfg(any(test, axtest))]
pub(crate) fn new() -> crate::StarryResult<Self> {
Self::from_tick_accounting(scheduler::runtime::service::SchedulerTickCpuTime::new())
}
pub(crate) fn with_realtime_gate(
gate: Arc<scheduler::runtime::service::SchedulerTickGate>,
) -> crate::StarryResult<Self> {
Self::from_tick_accounting(
scheduler::runtime::service::SchedulerTickCpuTime::with_realtime_gate(gate),
)
}
fn from_tick_accounting(
accounting: scheduler::runtime::service::SchedulerTickCpuTime,
) -> crate::StarryResult<Self> {
let scheduler_tick_cpu_time = crate::task::allocation::try_arc(accounting)?;
Ok(Self {
scheduler_tick_cpu_time,
published_user_ns: AtomicU64::new(0),
published_system_ns: AtomicU64::new(0),
published_runtime_ns: AtomicU64::new(0),
adjusted: RawSpinLock::new(CpuTimeHighWater::ZERO),
})
}
pub(crate) fn realtime_ticks(&self) -> Option<(u64, core::num::NonZeroU64)> {
self.scheduler_tick_cpu_time.realtime_ticks()
}
pub fn output(&self, runtime_ns: u64) -> (TimeValue, TimeValue) {
let snapshot = self.snapshot(runtime_ns);
let adjusted = adjust_cpu_time(
snapshot.raw_user_ns,
snapshot.raw_system_ns,
snapshot.runtime_ns,
&self.adjusted,
);
(
time_value_from_nanos(adjusted.user_ns),
time_value_from_nanos(adjusted.system_ns),
)
}
pub(crate) fn scheduler_tick_cpu_time(
&self,
) -> Arc<scheduler::runtime::service::SchedulerTickCpuTime> {
Arc::clone(&self.scheduler_tick_cpu_time)
}
pub(crate) fn sample_scheduler_tick(&self, runtime_ns: u64) -> CpuTimeDelta {
self.publish_snapshot_delta(self.snapshot(runtime_ns))
}
pub(crate) fn unpublished_delta(&self, runtime_ns: u64) -> CpuTimeDelta {
let snapshot = self.snapshot(runtime_ns);
CpuTimeDelta {
raw_user_ns: snapshot
.raw_user_ns
.saturating_sub(self.published_user_ns.load(Ordering::Acquire)),
raw_system_ns: snapshot
.raw_system_ns
.saturating_sub(self.published_system_ns.load(Ordering::Acquire)),
runtime_ns: snapshot
.runtime_ns
.saturating_sub(self.published_runtime_ns.load(Ordering::Acquire)),
}
}
pub(super) fn snapshot(&self, runtime_ns: u64) -> CpuTimeSnapshot {
let tick = self.scheduler_tick_cpu_time.snapshot();
CpuTimeSnapshot {
raw_user_ns: tick.user_ns(),
raw_system_ns: tick.system_ns(),
runtime_ns,
}
}
pub(crate) fn publish_committed_delta(&self, runtime_ns: u64) -> CpuTimeDelta {
let tick = self.scheduler_tick_cpu_time.snapshot();
self.publish_totals(tick.user_ns(), tick.system_ns(), runtime_ns)
}
pub(crate) fn published_runtime_ns(&self) -> u64 {
self.published_runtime_ns.load(Ordering::Acquire)
}
fn publish_snapshot_delta(&self, snapshot: CpuTimeSnapshot) -> CpuTimeDelta {
self.publish_totals(
snapshot.raw_user_ns,
snapshot.raw_system_ns,
snapshot.runtime_ns,
)
}
fn publish_totals(
&self,
raw_user_ns: u64,
raw_system_ns: u64,
runtime_ns: u64,
) -> CpuTimeDelta {
let previous_user = self
.published_user_ns
.fetch_max(raw_user_ns, Ordering::AcqRel);
let previous_system = self
.published_system_ns
.fetch_max(raw_system_ns, Ordering::AcqRel);
let previous_runtime = self
.published_runtime_ns
.fetch_max(runtime_ns, Ordering::AcqRel);
CpuTimeDelta {
raw_user_ns: raw_user_ns.saturating_sub(previous_user),
raw_system_ns: raw_system_ns.saturating_sub(previous_system),
runtime_ns: runtime_ns.saturating_sub(previous_runtime),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) struct CpuTimeSnapshot {
pub(super) raw_user_ns: u64,
pub(super) raw_system_ns: u64,
pub(super) runtime_ns: u64,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub(crate) struct CpuTimeDelta {
raw_user_ns: u64,
raw_system_ns: u64,
runtime_ns: u64,
}
impl CpuTimeDelta {
pub(crate) const ZERO: Self = Self {
raw_user_ns: 0,
raw_system_ns: 0,
runtime_ns: 0,
};
pub(crate) fn add(self, other: Self) -> Self {
Self {
raw_user_ns: self.raw_user_ns.saturating_add(other.raw_user_ns),
raw_system_ns: self.raw_system_ns.saturating_add(other.raw_system_ns),
runtime_ns: self.runtime_ns.saturating_add(other.runtime_ns),
}
}
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
struct CpuTimeHighWater {
user_ns: u64,
system_ns: u64,
}
impl CpuTimeHighWater {
const ZERO: Self = Self {
user_ns: 0,
system_ns: 0,
};
}
fn adjust_cpu_time(
raw_user_ns: u64,
raw_system_ns: u64,
runtime_ns: u64,
high_water: &RawSpinLock<CpuTimeHighWater>,
) -> CpuTimeHighWater {
let mut previous = high_water.lock();
if previous.user_ns.saturating_add(previous.system_ns) >= runtime_ns {
return *previous;
}
let mut system_ns = if raw_system_ns == 0 {
0
} else if raw_user_ns == 0 {
runtime_ns
} else {
let raw_total = raw_user_ns as u128 + raw_system_ns as u128;
((raw_system_ns as u128 * runtime_ns as u128) / raw_total) as u64
};
system_ns = system_ns.max(previous.system_ns).min(runtime_ns);
let mut user_ns = runtime_ns - system_ns;
if user_ns < previous.user_ns {
user_ns = previous.user_ns;
system_ns = runtime_ns - user_ns;
}
*previous = CpuTimeHighWater { user_ns, system_ns };
*previous
}
pub struct ProcessCpuTimeAccounting {
raw_user_ns: AtomicU64,
raw_system_ns: AtomicU64,
runtime_ns: AtomicU64,
adjusted: RawSpinLock<CpuTimeHighWater>,
#[cfg(axtest)]
publication_rmws: AtomicU64,
}
impl Default for ProcessCpuTimeAccounting {
fn default() -> Self {
Self::new()
}
}
impl ProcessCpuTimeAccounting {
pub(crate) const fn new() -> Self {
Self {
raw_user_ns: AtomicU64::new(0),
raw_system_ns: AtomicU64::new(0),
runtime_ns: AtomicU64::new(0),
adjusted: RawSpinLock::new(CpuTimeHighWater::ZERO),
#[cfg(axtest)]
publication_rmws: AtomicU64::new(0),
}
}
pub(crate) fn record_transition(&self, transition: impl FnOnce() -> CpuTimeDelta) {
self.record_delta(transition());
}
fn record_delta(&self, delta: CpuTimeDelta) {
if delta == CpuTimeDelta::ZERO {
return;
}
#[cfg(axtest)]
self.publication_rmws.fetch_add(3, Ordering::Relaxed);
self.raw_user_ns
.fetch_add(delta.raw_user_ns, Ordering::Release);
self.raw_system_ns
.fetch_add(delta.raw_system_ns, Ordering::Release);
self.runtime_ns
.fetch_add(delta.runtime_ns, Ordering::Release);
}
pub(crate) fn snapshot_with_live(
&self,
mut live_residual: impl FnMut(u64) -> CpuTimeDelta,
) -> ProcessCpuTimeSnapshot {
self.snapshot_at_with_live(monotonic_time_nanos() as u64, &mut live_residual)
}
pub(crate) fn snapshot_committed(&self) -> ProcessCpuTimeSnapshot {
self.snapshot_committed_at(monotonic_time_nanos() as u64)
}
fn snapshot_committed_at(&self, now_ns: u64) -> ProcessCpuTimeSnapshot {
self.snapshot_at_with_live(now_ns, &mut |_| CpuTimeDelta::ZERO)
}
fn snapshot_at_with_live(
&self,
now_ns: u64,
live_residual: &mut impl FnMut(u64) -> CpuTimeDelta,
) -> ProcessCpuTimeSnapshot {
let committed = CpuTimeDelta {
raw_user_ns: self.raw_user_ns.load(Ordering::Acquire),
raw_system_ns: self.raw_system_ns.load(Ordering::Acquire),
runtime_ns: self.runtime_ns.load(Ordering::Acquire),
};
let sampled = committed.add(live_residual(now_ns));
let adjusted = adjust_cpu_time(
sampled.raw_user_ns,
sampled.raw_system_ns,
sampled.runtime_ns,
&self.adjusted,
);
ProcessCpuTimeSnapshot {
user_ns: adjusted.user_ns,
system_ns: adjusted.system_ns,
sampled_at_ns: now_ns,
}
}
}
#[cfg(axtest)]
pub(super) fn zero_process_cpu_time_delta_avoids_publication_for_test() -> bool {
let accounting = ProcessCpuTimeAccounting::new();
accounting.record_transition(|| CpuTimeDelta::ZERO);
accounting.publication_rmws.load(Ordering::Relaxed) == 0
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) struct ProcessCpuTimeSnapshot {
pub(super) user_ns: u64,
pub(super) system_ns: u64,
pub(super) sampled_at_ns: u64,
}
impl ProcessCpuTimeSnapshot {
pub(crate) fn output(self) -> (TimeValue, TimeValue) {
(
time_value_from_nanos(self.user_ns),
time_value_from_nanos(self.system_ns),
)
}
}
#[cfg(axtest)]
pub(super) fn process_cpu_high_water_preserves_runtime_total_for_test() -> bool {
let process = ProcessCpuTimeAccounting::new();
let accounting = CpuTimeAccounting::new().unwrap();
let first =
process.snapshot_at_with_live(10, &mut |runtime| accounting.unpublished_delta(runtime));
process.record_transition(|| accounting.publish_committed_delta(15));
let second = process.snapshot_committed_at(15);
first.user_ns.saturating_add(first.system_ns) == 10
&& second.user_ns >= first.user_ns
&& second.system_ns >= first.system_ns
&& second.user_ns.saturating_add(second.system_ns) == 15
}
#[cfg(all(test, not(axtest)))]
include!("accounting/tests.rs");