use super::hrtick::finish_hrtick_delta_ns;
use crate::{
sched::{FairMode, Nice},
thread::TaskError,
};
const BASE_WEIGHT: u64 = 1024;
const MAX_VIRTUAL_DELTA: u64 = i64::MAX as u64;
pub(crate) const fn virtual_delta(value: u64, reference: u64) -> i64 {
value.wrapping_sub(reference) as i64
}
pub(crate) const fn virtual_before(value: u64, reference: u64) -> bool {
virtual_delta(value, reference) < 0
}
pub(crate) const fn virtual_after(value: u64, reference: u64) -> bool {
virtual_delta(value, reference) > 0
}
pub(crate) const fn virtual_min(lhs: u64, rhs: u64) -> u64 {
if virtual_before(rhs, lhs) { rhs } else { lhs }
}
pub(crate) const fn virtual_max(lhs: u64, rhs: u64) -> u64 {
if virtual_after(rhs, lhs) { rhs } else { lhs }
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) struct FairEntity {
nice: Nice,
mode: FairMode,
vruntime: u64,
service_request_ns: u64,
virtual_deadline: u64,
protected_until_vruntime: u64,
placement: FairPlacement,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum FairPlacement {
Initial,
Active,
Delayed {
virtual_lag: i64,
},
DelayedMigrating {
virtual_lag: i64,
relative_deadline: u64,
},
Sleeping {
virtual_lag: i64,
},
Migrating {
virtual_lag: i64,
relative_deadline: u64,
},
}
const fn load_weight(nice: Nice, mode: FairMode) -> u32 {
match mode {
FairMode::Idle => crate::sched::SchedulePolicy::IDLE_POLICY_WEIGHT,
FairMode::Normal | FairMode::Batch => nice.weight(),
}
}
impl FairEntity {
#[cfg(test)]
pub(super) const fn test_state(
nice: Nice,
mode: FairMode,
vruntime: u64,
virtual_deadline: u64,
) -> Self {
Self {
nice,
mode,
vruntime,
service_request_ns: 1,
virtual_deadline,
protected_until_vruntime: virtual_deadline,
placement: FairPlacement::Active,
}
}
pub fn new(nice: Nice, mode: FairMode, request_ns: u64, virtual_time: u64) -> Self {
let weighted_request = weighted_delta(request_ns, load_weight(nice, mode));
Self {
nice,
mode,
vruntime: virtual_time,
service_request_ns: request_ns,
virtual_deadline: virtual_time.wrapping_add(weighted_request),
protected_until_vruntime: virtual_time,
placement: FairPlacement::Initial,
}
}
pub fn charge(&mut self, runtime_ns: u64, _virtual_time: u64) -> bool {
self.vruntime = self
.vruntime
.wrapping_add(weighted_delta(runtime_ns, self.load_weight()));
let request_exhausted = self.request_exhausted();
if request_exhausted {
self.renew_request();
}
request_exhausted
}
pub(crate) fn place_at_least(&mut self, virtual_time: u64) {
if !virtual_before(self.vruntime, virtual_time) {
return;
}
let shift = virtual_time.wrapping_sub(self.vruntime);
self.vruntime = virtual_time;
self.virtual_deadline = self.virtual_deadline.wrapping_add(shift);
if self.slice_is_protected() {
self.protected_until_vruntime = self.protected_until_vruntime.wrapping_add(shift);
} else {
self.protected_until_vruntime = self.vruntime;
}
}
pub(crate) fn capture_sleep_lag(
&mut self,
virtual_time: u64,
rq_max_slice_ns: u64,
timing_granularity_ns: u64,
) {
let virtual_lag =
self.bounded_virtual_lag(virtual_time, rq_max_slice_ns, timing_granularity_ns);
#[cfg(feature = "qperf-metrics")]
crate::diagnostics::counters::record_fair_sleep_lag(virtual_lag);
self.placement = FairPlacement::Sleeping { virtual_lag };
}
pub(crate) fn capture_migration(
&mut self,
virtual_time: u64,
rq_max_slice_ns: u64,
timing_granularity_ns: u64,
) {
let relative_deadline = self.virtual_deadline.wrapping_sub(self.vruntime);
self.placement = match self.placement {
FairPlacement::Delayed { virtual_lag } => FairPlacement::DelayedMigrating {
virtual_lag: self.refreshed_delayed_lag(
virtual_time,
rq_max_slice_ns,
timing_granularity_ns,
virtual_lag,
),
relative_deadline,
},
FairPlacement::Migrating { .. } | FairPlacement::DelayedMigrating { .. } => return,
FairPlacement::Initial | FairPlacement::Active | FairPlacement::Sleeping { .. } => {
FairPlacement::Migrating {
virtual_lag: self.bounded_virtual_lag(
virtual_time,
rq_max_slice_ns,
timing_granularity_ns,
),
relative_deadline,
}
}
};
}
pub(crate) fn place_after_activation(
&mut self,
virtual_time: u64,
runnable_weight: u64,
) -> Result<(), TaskError> {
#[cfg(feature = "qperf-metrics")]
let sleeping = matches!(self.placement, FairPlacement::Sleeping { .. });
let (saved_lag, request_ns) = match self.placement {
FairPlacement::Initial => (0, self.service_request_ns / 2),
FairPlacement::Sleeping { virtual_lag } => (virtual_lag, self.service_request_ns),
FairPlacement::Active
| FairPlacement::Delayed { .. }
| FairPlacement::DelayedMigrating { .. }
| FairPlacement::Migrating { .. } => {
return Err(TaskError::InvalidConfiguration);
}
};
#[cfg(feature = "qperf-metrics")]
if sleeping {
crate::diagnostics::counters::record_fair_sleep_wake_lag(saved_lag);
}
let placement_lag = self.inflated_placement_lag(saved_lag, runnable_weight);
self.vruntime = virtual_time.wrapping_sub(placement_lag as u64);
self.virtual_deadline = self
.vruntime
.wrapping_add(weighted_delta(request_ns, self.load_weight()));
self.protected_until_vruntime = self.vruntime;
self.placement = FairPlacement::Active;
Ok(())
}
pub(crate) fn place_after_transfer(
&mut self,
virtual_time: u64,
runnable_weight: u64,
) -> Result<(), TaskError> {
match self.placement {
FairPlacement::Initial | FairPlacement::Sleeping { .. } => {
self.place_after_activation(virtual_time, runnable_weight)
}
FairPlacement::Migrating {
virtual_lag,
relative_deadline,
} => {
let placement_lag = self.inflated_placement_lag(virtual_lag, runnable_weight);
self.vruntime = virtual_time.wrapping_sub(placement_lag as u64);
self.virtual_deadline = self.vruntime.wrapping_add(relative_deadline);
self.protected_until_vruntime = self.vruntime;
self.placement = FairPlacement::Active;
Ok(())
}
FairPlacement::Delayed { .. } | FairPlacement::DelayedMigrating { .. } => {
Err(TaskError::InvalidConfiguration)
}
FairPlacement::Active => {
self.place_at_least(virtual_time);
self.cancel_slice_protection();
Ok(())
}
}
}
fn bounded_virtual_lag(
self,
virtual_time: u64,
rq_max_slice_ns: u64,
timing_granularity_ns: u64,
) -> i64 {
let limit = weighted_delta(
rq_max_slice_ns.saturating_add(timing_granularity_ns),
self.load_weight(),
)
.min(i64::MAX as u64) as i64;
virtual_delta(virtual_time, self.vruntime).clamp(-limit, limit)
}
pub(crate) fn begin_delayed_dequeue(
&mut self,
virtual_time: u64,
rq_max_slice_ns: u64,
timing_granularity_ns: u64,
) {
let virtual_lag = self
.bounded_virtual_lag(virtual_time, rq_max_slice_ns, timing_granularity_ns)
.min(0);
#[cfg(feature = "qperf-metrics")]
crate::diagnostics::counters::record_fair_delayed_begin(virtual_lag);
self.placement = FairPlacement::Delayed { virtual_lag };
}
pub(crate) const fn is_delayed(self) -> bool {
matches!(self.placement, FairPlacement::Delayed { .. })
}
pub(crate) const fn is_delayed_migrating(self) -> bool {
matches!(self.placement, FairPlacement::DelayedMigrating { .. })
}
fn refreshed_delayed_lag(
self,
virtual_time: u64,
rq_max_slice_ns: u64,
timing_granularity_ns: u64,
saved_lag: i64,
) -> i64 {
self.bounded_virtual_lag(virtual_time, rq_max_slice_ns, timing_granularity_ns)
.max(saved_lag)
.min(0)
}
pub(crate) fn delayed_requeue_lag(
self,
virtual_time: u64,
rq_max_slice_ns: u64,
timing_granularity_ns: u64,
) -> Result<Option<i64>, TaskError> {
let FairPlacement::Delayed {
virtual_lag: saved_lag,
} = self.placement
else {
return Err(TaskError::InvalidConfiguration);
};
#[cfg(feature = "qperf-metrics")]
crate::diagnostics::counters::record_fair_delayed_wake_refresh(
saved_lag,
self.bounded_virtual_lag(virtual_time, rq_max_slice_ns, timing_granularity_ns),
);
let lag = self.refreshed_delayed_lag(
virtual_time,
rq_max_slice_ns,
timing_granularity_ns,
saved_lag,
);
#[cfg(feature = "qperf-metrics")]
crate::diagnostics::counters::record_fair_delayed_wake_lag(lag);
let placed_vruntime = virtual_time.wrapping_sub(lag as u64);
Ok((placed_vruntime != self.vruntime).then_some(lag))
}
pub(crate) fn clear_delayed(&mut self) -> Result<(), TaskError> {
if !matches!(self.placement, FairPlacement::Delayed { .. }) {
return Err(TaskError::InvalidConfiguration);
}
self.placement = FairPlacement::Active;
Ok(())
}
pub(crate) fn place_reactivated_delayed(
&mut self,
virtual_time: u64,
runnable_weight: u64,
saved_lag: i64,
) -> Result<(), TaskError> {
if !matches!(self.placement, FairPlacement::Delayed { .. }) {
return Err(TaskError::InvalidConfiguration);
}
let placement_lag = self.inflated_placement_lag(saved_lag, runnable_weight);
self.vruntime = virtual_time.wrapping_sub(placement_lag as u64);
self.virtual_deadline = self
.vruntime
.wrapping_add(weighted_delta(self.service_request_ns, self.load_weight()));
self.protected_until_vruntime = self.vruntime;
self.placement = FairPlacement::Active;
Ok(())
}
pub(crate) fn finish_delayed_dequeue(
&mut self,
virtual_time: u64,
rq_max_slice_ns: u64,
timing_granularity_ns: u64,
) -> Result<(), TaskError> {
let FairPlacement::Delayed {
virtual_lag: saved_lag,
} = self.placement
else {
return Err(TaskError::InvalidConfiguration);
};
let virtual_lag = self.refreshed_delayed_lag(
virtual_time,
rq_max_slice_ns,
timing_granularity_ns,
saved_lag,
);
self.placement = FairPlacement::Sleeping { virtual_lag };
Ok(())
}
pub(crate) fn place_delayed_after_transfer(
&mut self,
virtual_time: u64,
runnable_weight: u64,
) -> Result<(), TaskError> {
let FairPlacement::DelayedMigrating {
virtual_lag,
relative_deadline,
} = self.placement
else {
return Err(TaskError::InvalidConfiguration);
};
let placement_lag = self.inflated_placement_lag(virtual_lag, runnable_weight);
self.vruntime = virtual_time.wrapping_sub(placement_lag as u64);
self.virtual_deadline = self.vruntime.wrapping_add(relative_deadline);
self.protected_until_vruntime = self.vruntime;
self.placement = FairPlacement::Delayed { virtual_lag };
Ok(())
}
fn inflated_placement_lag(self, saved_lag: i64, runnable_weight: u64) -> i64 {
if saved_lag == 0 || runnable_weight == 0 {
return 0;
}
let weight = u64::from(self.load_weight());
(i128::from(saved_lag).saturating_mul(i128::from(runnable_weight.saturating_add(weight)))
/ i128::from(runnable_weight))
.clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
}
pub(crate) fn renew_request(&mut self) {
self.virtual_deadline = self
.vruntime
.wrapping_add(weighted_delta(self.service_request_ns, self.load_weight()));
self.protected_until_vruntime = self.vruntime;
}
pub(crate) fn yield_request(&mut self, virtual_time: u64) {
let eligible = self.is_eligible(virtual_time);
#[cfg(feature = "qperf-metrics")]
crate::diagnostics::counters::record_fair_yield(
eligible,
if eligible {
virtual_delta(self.virtual_deadline, self.vruntime).max(0) as u64
} else {
0
},
if eligible {
0
} else {
virtual_delta(self.vruntime, virtual_time).max(0) as u64
},
);
if eligible {
self.vruntime = virtual_max(self.vruntime, self.virtual_deadline);
self.renew_request();
} else if self.request_exhausted() {
self.renew_request();
}
}
pub(crate) fn reconfigure(
mut self,
nice: Nice,
mode: FairMode,
source_virtual_time: u64,
destination_virtual_time: u64,
) -> Self {
let old_weight = self.load_weight();
let new_weight = load_weight(nice, mode);
let reweight_lag = |lag: i64| {
(i128::from(lag) * i128::from(old_weight) / i128::from(new_weight))
.clamp(-i128::from(i64::MAX), i128::from(i64::MAX)) as i64
};
let protected = self.slice_is_protected();
let relative_protection = i128::from(virtual_delta(
self.protected_until_vruntime,
source_virtual_time,
));
let relative_deadline =
i128::from(virtual_delta(self.virtual_deadline, source_virtual_time));
let lag = i128::from(virtual_delta(source_virtual_time, self.vruntime));
let reweighted_lag = reweight_lag(lag as i64);
let reweighted_deadline =
(relative_deadline * i128::from(old_weight) / i128::from(new_weight))
.clamp(-i128::from(i64::MAX), i128::from(i64::MAX)) as i64;
self.nice = nice;
self.mode = mode;
self.vruntime = destination_virtual_time.wrapping_sub(reweighted_lag as u64);
self.virtual_deadline = destination_virtual_time.wrapping_add(reweighted_deadline as u64);
let active_relative_deadline = self.virtual_deadline.wrapping_sub(self.vruntime);
self.protected_until_vruntime = if protected {
let reweighted_protection =
(relative_protection * i128::from(old_weight) / i128::from(new_weight))
.clamp(-i128::from(i64::MAX), i128::from(i64::MAX)) as i64;
destination_virtual_time.wrapping_add(reweighted_protection as u64)
} else {
self.vruntime
};
self.placement = match self.placement {
FairPlacement::Sleeping { virtual_lag } => FairPlacement::Sleeping {
virtual_lag: reweight_lag(virtual_lag),
},
FairPlacement::Delayed { virtual_lag } => FairPlacement::Delayed {
virtual_lag: reweight_lag(virtual_lag),
},
FairPlacement::Migrating { virtual_lag, .. } => FairPlacement::Migrating {
virtual_lag: reweight_lag(virtual_lag),
relative_deadline: active_relative_deadline,
},
FairPlacement::DelayedMigrating { virtual_lag, .. } => {
FairPlacement::DelayedMigrating {
virtual_lag: reweight_lag(virtual_lag),
relative_deadline: active_relative_deadline,
}
}
FairPlacement::Initial | FairPlacement::Active => self.placement,
};
self
}
pub(crate) fn set_slice_protection(&mut self, shortest_competing_slice_ns: Option<u64>) {
let protected_slice_ns = shortest_competing_slice_ns
.unwrap_or(self.service_request_ns)
.min(self.service_request_ns);
self.protected_until_vruntime = if protected_slice_ns == self.service_request_ns {
self.virtual_deadline
} else {
virtual_min(
self.virtual_deadline,
self.vruntime
.wrapping_add(weighted_delta(protected_slice_ns, self.load_weight())),
)
};
}
pub(crate) fn update_slice_protection(&mut self, shortest_queued_slice_ns: u64) {
let queued_boundary = self
.vruntime
.wrapping_add(weighted_delta(shortest_queued_slice_ns, self.load_weight()));
self.protected_until_vruntime = virtual_min(self.protected_until_vruntime, queued_boundary);
}
pub(crate) fn cancel_slice_protection(&mut self) {
self.protected_until_vruntime = self.vruntime;
}
pub(crate) const fn slice_is_protected(self) -> bool {
virtual_before(self.vruntime, self.protected_until_vruntime)
}
pub(crate) const fn has_shorter_slice_than(self, current: Self) -> bool {
self.service_request_ns < current.service_request_ns
}
pub(crate) fn runtime_deadline_delta_ns(self) -> u64 {
let virtual_delta = self.virtual_deadline.wrapping_sub(self.vruntime);
((u128::from(self.load_weight()) * u128::from(virtual_delta)) / u128::from(BASE_WEIGHT))
.min(u128::from(u64::MAX)) as u64
}
pub(crate) fn finish_runtime_deadline_delta_ns(self, irq_util_avg: u32) -> u64 {
finish_hrtick_delta_ns(self.runtime_deadline_delta_ns(), irq_util_avg)
}
}
impl FairEntity {
pub(crate) const fn request_exhausted(self) -> bool {
!virtual_before(self.vruntime, self.virtual_deadline)
}
pub const fn is_eligible(self, virtual_time: u64) -> bool {
!virtual_after(self.vruntime, virtual_time)
}
pub const fn mode(self) -> FairMode {
self.mode
}
pub const fn vruntime(self) -> u64 {
self.vruntime
}
pub(crate) const fn weight(self) -> u32 {
self.load_weight()
}
const fn load_weight(self) -> u32 {
load_weight(self.nice, self.mode)
}
pub const fn virtual_deadline(self) -> u64 {
self.virtual_deadline
}
pub(crate) const fn deadline_precedes(self, current: Self) -> bool {
virtual_before(self.virtual_deadline, current.virtual_deadline)
}
pub(crate) const fn service_request_ns(self) -> u64 {
self.service_request_ns
}
}
const fn weighted_delta_needs_scaling(weight: u32) -> bool {
weight != BASE_WEIGHT as u32
}
fn weighted_delta(runtime_ns: u64, weight: u32) -> u64 {
if !weighted_delta_needs_scaling(weight) {
return runtime_ns.min(MAX_VIRTUAL_DELTA);
}
((runtime_ns as u128).saturating_mul(BASE_WEIGHT as u128) / weight as u128)
.min(MAX_VIRTUAL_DELTA as u128) as u64
}
#[cfg(test)]
mod tests;