#[cfg(feature = "sched-isr-timing")]
use core::ptr;
use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering};
use crate::thread::ThreadCtx;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TraceThread {
pub id: u32,
pub name: &'static str,
pub is_cfs: bool,
}
impl TraceThread {
fn from_ptr(thread: *const ThreadCtx) -> Option<Self> {
if thread.is_null() {
return None;
}
let thread = unsafe { &*thread };
Some(Self {
id: thread.id,
name: thread.name,
is_cfs: thread.is_cfs(),
})
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TraceEvent {
ContextSwitch {
from: Option<TraceThread>,
to: TraceThread,
},
DeadlineMiss {
thread: TraceThread,
runtime_ticks: u32,
relative_deadline_ticks: u32,
},
Wakeup {
thread: TraceThread,
},
Yield {
thread: TraceThread,
elapsed_ticks: u32,
},
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct TraceCounters {
pub context_switches: u32,
pub deadline_misses: u32,
pub wakeups: u32,
pub yields: u32,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct SchedIsrTiming {
pub samples: u32,
pub last_ticks: u32,
pub min_ticks: u32,
pub max_ticks: u32,
pub advance_ktimers_last_ticks: u32,
pub advance_ktimers_max_ticks: u32,
pub dispatch_expired_ktimer_last_ticks: u32,
pub dispatch_expired_ktimer_max_ticks: u32,
}
pub type TraceFn = fn(TraceEvent);
static TRACE_FN: AtomicUsize = AtomicUsize::new(0);
static CONTEXT_SWITCHES: AtomicU32 = AtomicU32::new(0);
static DEADLINE_MISSES: AtomicU32 = AtomicU32::new(0);
static WAKEUPS: AtomicU32 = AtomicU32::new(0);
static YIELDS: AtomicU32 = AtomicU32::new(0);
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_TO_PENDSV_ENABLED: u32 = 0;
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_TO_PENDSV_ARMED: u32 = 0;
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_TO_PENDSV_START_CYCLE: u32 = 0;
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_TO_PENDSV_LAST_TICKS: u32 = 0;
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_TO_PENDSV_MIN_TICKS: u32 = 0;
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_TO_PENDSV_MAX_TICKS: u32 = 0;
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_TO_PENDSV_SAMPLES: u32 = 0;
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_ADVANCE_KTIMERS_LAST_TICKS: u32 = 0;
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_ADVANCE_KTIMERS_MAX_TICKS: u32 = 0;
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_DISPATCH_EXPIRED_KTIMER_LAST_TICKS: u32 = 0;
#[cfg(feature = "sched-isr-timing")]
#[unsafe(no_mangle)]
pub static mut SCHED_TICK_DISPATCH_EXPIRED_KTIMER_MAX_TICKS: u32 = 0;
pub fn set_trace_fn(trace_fn: TraceFn) {
TRACE_FN.store(trace_fn as usize, Ordering::Release);
}
pub fn clear_trace_fn() {
TRACE_FN.store(0, Ordering::Release);
}
pub fn trace_counters() -> TraceCounters {
TraceCounters {
context_switches: CONTEXT_SWITCHES.load(Ordering::Relaxed),
deadline_misses: DEADLINE_MISSES.load(Ordering::Relaxed),
wakeups: WAKEUPS.load(Ordering::Relaxed),
yields: YIELDS.load(Ordering::Relaxed),
}
}
pub fn reset_trace_counters() {
CONTEXT_SWITCHES.store(0, Ordering::Relaxed);
DEADLINE_MISSES.store(0, Ordering::Relaxed);
WAKEUPS.store(0, Ordering::Relaxed);
YIELDS.store(0, Ordering::Relaxed);
}
#[inline]
pub fn mark_sched_tick_to_pendsv_start() {
#[cfg(feature = "sched-isr-timing")]
{
let start_cycle = crate::arch::platform::dwt_cycle_count();
unsafe {
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_ENABLED, 1);
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_START_CYCLE, start_cycle);
}
}
}
#[inline]
pub(crate) fn arm_sched_tick_to_pendsv_sample() {
#[cfg(feature = "sched-isr-timing")]
{
unsafe {
if ptr::read_volatile(&raw const SCHED_TICK_TO_PENDSV_ENABLED) != 0 {
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_ARMED, 1);
}
}
}
}
#[inline]
#[cfg(feature = "sched-isr-timing")]
#[allow(unused)]
pub(crate) fn record_sched_tick_ktimer_timing(
advance_ktimers_ticks: u32,
dispatch_expired_ktimer_ticks: u32,
) {
unsafe {
record_last_and_max(
&raw mut SCHED_TICK_ADVANCE_KTIMERS_LAST_TICKS,
&raw mut SCHED_TICK_ADVANCE_KTIMERS_MAX_TICKS,
advance_ktimers_ticks,
);
record_last_and_max(
&raw mut SCHED_TICK_DISPATCH_EXPIRED_KTIMER_LAST_TICKS,
&raw mut SCHED_TICK_DISPATCH_EXPIRED_KTIMER_MAX_TICKS,
dispatch_expired_ktimer_ticks,
);
}
}
#[inline]
pub fn sched_tick_to_pendsv_timing() -> SchedIsrTiming {
#[cfg(feature = "sched-isr-timing")]
{
unsafe {
return SchedIsrTiming {
samples: ptr::read_volatile(&raw const SCHED_TICK_TO_PENDSV_SAMPLES),
last_ticks: ptr::read_volatile(&raw const SCHED_TICK_TO_PENDSV_LAST_TICKS),
min_ticks: ptr::read_volatile(&raw const SCHED_TICK_TO_PENDSV_MIN_TICKS),
max_ticks: ptr::read_volatile(&raw const SCHED_TICK_TO_PENDSV_MAX_TICKS),
advance_ktimers_last_ticks: ptr::read_volatile(
&raw const SCHED_TICK_ADVANCE_KTIMERS_LAST_TICKS,
),
advance_ktimers_max_ticks: ptr::read_volatile(
&raw const SCHED_TICK_ADVANCE_KTIMERS_MAX_TICKS,
),
dispatch_expired_ktimer_last_ticks: ptr::read_volatile(
&raw const SCHED_TICK_DISPATCH_EXPIRED_KTIMER_LAST_TICKS,
),
dispatch_expired_ktimer_max_ticks: ptr::read_volatile(
&raw const SCHED_TICK_DISPATCH_EXPIRED_KTIMER_MAX_TICKS,
),
};
}
}
#[cfg(not(feature = "sched-isr-timing"))]
{
SchedIsrTiming::default()
}
}
#[inline]
pub fn reset_sched_tick_to_pendsv_timing() {
#[cfg(feature = "sched-isr-timing")]
{
unsafe {
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_ARMED, 0);
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_START_CYCLE, 0);
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_LAST_TICKS, 0);
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_MIN_TICKS, 0);
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_MAX_TICKS, 0);
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_SAMPLES, 0);
ptr::write_volatile(&raw mut SCHED_TICK_ADVANCE_KTIMERS_LAST_TICKS, 0);
ptr::write_volatile(&raw mut SCHED_TICK_ADVANCE_KTIMERS_MAX_TICKS, 0);
ptr::write_volatile(&raw mut SCHED_TICK_DISPATCH_EXPIRED_KTIMER_LAST_TICKS, 0);
ptr::write_volatile(&raw mut SCHED_TICK_DISPATCH_EXPIRED_KTIMER_MAX_TICKS, 0);
}
}
}
#[inline]
pub fn reset_sched_tick_to_pendsv_min_ticks() {
#[cfg(feature = "sched-isr-timing")]
{
crate::critical_section(|| unsafe {
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_MIN_TICKS, 0);
});
}
}
#[inline]
pub fn reset_sched_tick_to_pendsv_max_ticks() {
#[cfg(feature = "sched-isr-timing")]
{
crate::critical_section(|| unsafe {
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_MAX_TICKS, 0);
});
}
}
#[inline]
pub fn reset_sched_tick_to_pendsv_min_max_ticks() {
#[cfg(feature = "sched-isr-timing")]
{
crate::critical_section(|| unsafe {
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_MIN_TICKS, 0);
ptr::write_volatile(&raw mut SCHED_TICK_TO_PENDSV_MAX_TICKS, 0);
});
}
}
pub(crate) fn record_context_switch(from: *const ThreadCtx, to: *const ThreadCtx) {
if from == to {
return;
}
let Some(to) = TraceThread::from_ptr(to) else {
return;
};
increment(&CONTEXT_SWITCHES);
emit(TraceEvent::ContextSwitch {
from: TraceThread::from_ptr(from),
to,
});
}
pub(crate) fn record_deadline_miss(
thread: *const ThreadCtx,
runtime_ticks: u32,
relative_deadline_ticks: u32,
) {
let Some(thread) = TraceThread::from_ptr(thread) else {
return;
};
increment(&DEADLINE_MISSES);
emit(TraceEvent::DeadlineMiss {
thread,
runtime_ticks,
relative_deadline_ticks,
});
}
pub(crate) fn record_wakeup(thread: *const ThreadCtx) {
let Some(thread) = TraceThread::from_ptr(thread) else {
return;
};
increment(&WAKEUPS);
emit(TraceEvent::Wakeup { thread });
}
pub(crate) fn record_yield(thread: *const ThreadCtx, elapsed_ticks: u32) {
let Some(thread) = TraceThread::from_ptr(thread) else {
return;
};
increment(&YIELDS);
emit(TraceEvent::Yield {
thread,
elapsed_ticks,
});
}
fn increment(counter: &AtomicU32) {
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| {
value.checked_add(1)
});
}
#[cfg(feature = "sched-isr-timing")]
unsafe fn record_last_and_max(last: *mut u32, max: *mut u32, ticks: u32) {
unsafe {
ptr::write_volatile(last, ticks);
if ptr::read_volatile(max) < ticks {
ptr::write_volatile(max, ticks);
}
}
}
fn emit(event: TraceEvent) {
let trace_fn = TRACE_FN.load(Ordering::Acquire);
if trace_fn != 0 {
let trace_fn: TraceFn = unsafe { core::mem::transmute(trace_fn) };
trace_fn(event);
}
}