use crate::arena::{Arena, ArenaStats, DEFAULT_ARENA};
use crate::config::{ConfigError, PlatformLimits, SchedulerConfig, Slice};
use crate::critical;
use crate::ring;
use crate::tcb::{KernelConfig, TaskControlBlock, TaskState, KERNEL};
use core::cell::UnsafeCell;
use core::ptr;
struct ArenaCell(UnsafeCell<Arena>);
unsafe impl Sync for ArenaCell {}
static ARENA: ArenaCell = ArenaCell(UnsafeCell::new(Arena::empty()));
#[inline]
pub(crate) unsafe fn arena() -> *mut Arena {
ARENA.0.get()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SchedulerStats {
pub total_threads: usize,
pub active_threads: usize,
pub ticks: u64,
pub switches: u64,
pub ticks_deferred: u64,
pub worst_latency: u32,
pub last_latency: u32,
pub worst_period_error_ns: u32,
pub last_period_error_ns: u32,
pub slice_ns: u64,
pub slice_cycles: u32,
pub timer_hz: u32,
pub reclaimed: u64,
pub blocks_without_current: u64,
pub arena: ArenaStats,
pub running: bool,
}
pub fn init() {
if let Err(e) = init_with(SchedulerConfig::default()) {
panic!("rrkernel: scheduler_init() failed: {}", e);
}
}
pub fn init_with(cfg: SchedulerConfig) -> Result<(), ConfigError> {
let plan = crate::arch::plan_timer(&cfg)?;
if plan.slice_ns == 0 || plan.slice_cycles == 0 {
return Err(ConfigError::ZeroSlice);
}
let (base, len) = match cfg.arena {
Some(region) => (region.as_mut_ptr(), region.len()),
None => (DEFAULT_ARENA.as_ptr(), crate::arena::DEFAULT_ARENA_SIZE),
};
let min_arena = core::mem::size_of::<TaskControlBlock>() * 4 + 512;
if len < min_arena {
return Err(ConfigError::ArenaTooSmall {
provided: len,
minimum: min_arena,
});
}
let g = critical::enter();
unsafe {
if *KERNEL.running.get() {
return Err(ConfigError::AlreadyInitialized);
}
(*arena()).init(base, len);
*KERNEL.config.get() = KernelConfig {
stack_size: cfg.stack_size,
idle: cfg.idle,
measure: cfg.measure,
slice_cycles: plan.slice_cycles,
timer_hz: plan.timer_hz,
active_cores: (*KERNEL.config.get()).active_cores.max(1),
};
let tcb = alloc_tcb().ok_or(ConfigError::ArenaTooSmall {
provided: len,
minimum: min_arena,
})?;
(*tcb).state = TaskState::Running;
(*tcb).id = take_id();
crate::arch::adopt_current_task(tcb)?;
ring::insert_after(ptr::null_mut(), tcb);
KERNEL.set_current(tcb);
*KERNEL.ring_head.get() = tcb;
*KERNEL.total_threads.get() = 1;
*KERNEL.active_threads.get() = 1;
}
drop(g);
crate::arch::start_timer(plan)?;
let g = critical::enter();
unsafe { *KERNEL.running.get() = true };
drop(g);
Ok(())
}
pub fn set_slice(slice: Slice) -> Result<(), ConfigError> {
let g = critical::enter();
let timer_hz = unsafe {
if !*KERNEL.running.get() {
return Err(ConfigError::NotInitialized);
}
(*KERNEL.config.get()).timer_hz
};
drop(g);
let cfg = SchedulerConfig {
slice,
timer_hz,
..SchedulerConfig::default()
};
let plan = crate::arch::plan_timer(&cfg)?;
crate::arch::retune_timer(plan)?;
let g = critical::enter();
unsafe {
(*KERNEL.config.get()).slice_cycles = plan.slice_cycles;
(*KERNEL.config.get()).timer_hz = plan.timer_hz;
}
drop(g);
Ok(())
}
pub fn platform_limits() -> PlatformLimits {
crate::arch::platform_limits()
}
pub fn config() -> KernelConfig {
let g = critical::enter();
let c = unsafe { *KERNEL.config.get() };
drop(g);
c
}
pub fn slice_ns() -> u64 {
let cycles = config().slice_cycles;
crate::arch::cycles_to_ns(cycles)
}
pub fn stats() -> SchedulerStats {
let g = critical::enter();
let out = unsafe {
let cfg = *KERNEL.config.get();
SchedulerStats {
total_threads: *KERNEL.total_threads.get(),
active_threads: *KERNEL.active_threads.get(),
ticks: *KERNEL.ticks.get(),
switches: *KERNEL.switches.get(),
ticks_deferred: *KERNEL.ticks_deferred.get(),
worst_latency: *KERNEL.worst_latency.get(),
last_latency: *KERNEL.last_latency.get(),
worst_period_error_ns: *KERNEL.worst_period_error_ns.get(),
last_period_error_ns: *KERNEL.last_period_error_ns.get(),
slice_ns: crate::arch::cycles_to_ns(cfg.slice_cycles),
slice_cycles: cfg.slice_cycles,
timer_hz: cfg.timer_hz,
reclaimed: *KERNEL.reclaimed.get(),
blocks_without_current: *KERNEL.blocks_without_current.get(),
arena: (*arena()).stats(),
running: *KERNEL.running.get(),
}
};
drop(g);
out
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TaskInfo {
pub id: u32,
pub state: TaskState,
pub slices_run: u32,
pub switches: u32,
pub stack_size: usize,
pub is_current: bool,
}
pub fn for_each_task(mut f: impl FnMut(TaskInfo)) {
let g = critical::enter();
unsafe {
let cur = KERNEL.current();
let start = if cur.is_null() || !(*cur).is_linked() {
*KERNEL.ring_head.get()
} else {
cur
};
if !start.is_null() && (*start).is_linked() {
let mut p = start;
loop {
f(TaskInfo {
id: (*p).id,
state: (*p).state,
slices_run: (*p).slices_run,
switches: (*p).switches,
stack_size: (*p).stack_size,
is_current: p == cur,
});
p = (*p).next;
if p == start || p.is_null() {
break;
}
}
}
}
drop(g);
}
pub fn active_threads() -> usize {
let g = critical::enter();
let n = unsafe { *KERNEL.active_threads.get() };
drop(g);
n
}
pub fn total_threads() -> usize {
let g = critical::enter();
let n = unsafe { *KERNEL.total_threads.get() };
drop(g);
n
}
pub fn current_task_id() -> u32 {
let g = critical::enter();
let id = unsafe {
let c = KERNEL.current();
if c.is_null() {
0
} else {
(*c).id
}
};
drop(g);
id
}
pub fn current_slices_run() -> u32 {
let g = critical::enter();
let n = unsafe {
let c = KERNEL.current();
if c.is_null() {
0
} else {
(*c).slices_run
}
};
drop(g);
n
}
pub fn switch_count() -> u64 {
let g = critical::enter();
let n = unsafe { *KERNEL.switches.get() };
drop(g);
n
}
pub fn tick_count() -> u64 {
let g = critical::enter();
let n = unsafe { *KERNEL.ticks.get() };
drop(g);
n
}
pub fn shutdown(code: i32) -> ! {
let g = critical::enter();
unsafe { *KERNEL.shutdown.get() = true };
drop(g);
crate::arch::shutdown(code)
}
pub fn request_shutdown() {
let g = critical::enter();
unsafe { *KERNEL.shutdown.get() = true };
drop(g);
crate::arch::request_switch();
}
pub fn shutdown_requested() -> bool {
let g = critical::enter();
let b = unsafe { *KERNEL.shutdown.get() };
drop(g);
b
}
pub fn main_body<F: FnOnce()>(f: F) -> ! {
let me = {
let g = critical::enter();
let me = unsafe { KERNEL.current() };
drop(g);
assert!(
!me.is_null(),
"scheduler::main_body() requires scheduler::init*() to have run first"
);
me
};
f();
unsafe { crate::trampoline::exit_task(me) }
}
pub unsafe fn schedule_next() -> *mut TaskControlBlock {
let mut cur = KERNEL.current();
if !cur.is_null() && !(*cur).is_linked() {
cur = ptr::null_mut();
KERNEL.set_current(ptr::null_mut());
}
let next = if cur.is_null() {
crate::ring::first_runnable_from(*KERNEL.ring_head.get())
} else {
ring::next_runnable(cur)
};
if next.is_null() || !(*next).is_linked() {
return ptr::null_mut();
}
if next != cur {
if !cur.is_null() && (*cur).state == TaskState::Running {
(*cur).state = TaskState::Ready;
}
(*next).state = TaskState::Running;
KERNEL.set_current(next);
*KERNEL.switches.get() += 1;
}
(*next).switches = (*next).switches.wrapping_add(1);
(*next).slices_run = (*next).slices_run.wrapping_add(1);
next
}
#[inline]
pub unsafe fn on_tick() {
*KERNEL.ticks.get() += 1;
wake_expired();
}
#[inline]
pub unsafe fn on_tick_deferred() {
*KERNEL.ticks_deferred.get() += 1;
}
#[inline]
pub unsafe fn record_latency(value: u32) {
*KERNEL.last_latency.get() = value;
if value > *KERNEL.worst_latency.get() {
*KERNEL.worst_latency.get() = value;
}
}
#[inline]
pub unsafe fn record_period_error(value: u32) {
*KERNEL.last_period_error_ns.get() = value;
if value > *KERNEL.worst_period_error_ns.get() {
*KERNEL.worst_period_error_ns.get() = value;
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BlockOutcome {
Ready,
Blocked,
AlreadyPast,
NoCurrentTask,
}
pub unsafe fn mark_blocked_locked(resource: u32, deadline_tick: u64) -> BlockOutcome {
let me = KERNEL.current();
if me.is_null() {
*KERNEL.blocks_without_current.get() += 1;
debug_assert!(
false,
"rrkernel: blocking wait attempted with no current task (interrupt or idle context)"
);
return BlockOutcome::NoCurrentTask;
}
if deadline_tick != 0 && *KERNEL.ticks.get() >= deadline_tick {
return BlockOutcome::AlreadyPast;
}
(*me).blocked_on = resource;
(*me).block_deadline = deadline_tick;
(*me).state = TaskState::Blocked;
crate::arch::request_switch();
BlockOutcome::Blocked
}
pub fn block_until_tick(resource: u32, deadline_tick: u64) -> BlockOutcome {
let g = critical::enter();
let outcome = unsafe { mark_blocked_locked(resource, deadline_tick) };
drop(g);
outcome
}
pub fn block_until_tick_if(
resource: u32,
deadline_tick: u64,
ready: impl FnOnce() -> bool,
) -> BlockOutcome {
let g = critical::enter();
let outcome = if ready() {
BlockOutcome::Ready
} else {
unsafe { mark_blocked_locked(resource, deadline_tick) }
};
drop(g);
outcome
}
pub fn wake_task(id: u32) {
if id == 0 {
return;
}
let g = critical::enter();
unsafe {
let head = *KERNEL.ring_head.get();
if !head.is_null() {
let mut p = head;
loop {
if (*p).id == id {
(*p).flags |= crate::tcb::TCB_FLAG_WOKEN;
if (*p).state == TaskState::Blocked {
(*p).state = TaskState::Ready;
(*p).blocked_on = 0;
(*p).block_deadline = 0;
}
break;
}
p = (*p).next;
if p.is_null() || p == head {
break;
}
}
}
}
drop(g);
}
pub unsafe fn take_wake_flag_locked() -> bool {
let me = KERNEL.current();
if me.is_null() {
return false;
}
let set = (*me).flags & crate::tcb::TCB_FLAG_WOKEN != 0;
(*me).flags &= !crate::tcb::TCB_FLAG_WOKEN;
set
}
pub fn clear_timer_deadline() {
let g = critical::enter();
unsafe {
let me = KERNEL.current();
if !me.is_null() {
(*me).block_deadline = 0;
}
}
drop(g);
}
pub fn set_timer_deadline(deadline_tick: u64) {
if deadline_tick == 0 {
return;
}
let g = critical::enter();
unsafe {
let me = KERNEL.current();
if !me.is_null() {
let cur = (*me).block_deadline;
if cur == 0 || deadline_tick < cur {
(*me).block_deadline = deadline_tick;
}
}
}
drop(g);
}
pub fn task_timer_deadline() -> u64 {
let g = critical::enter();
let d = unsafe {
let me = KERNEL.current();
if me.is_null() {
0
} else {
(*me).block_deadline
}
};
drop(g);
d
}
pub fn clear_wake_flag() {
let g = critical::enter();
unsafe {
let me = KERNEL.current();
if !me.is_null() {
(*me).flags &= !crate::tcb::TCB_FLAG_WOKEN;
}
}
drop(g);
}
pub fn wake_first_blocked_on(resource: u32) -> bool {
let g = critical::enter();
let mut found = false;
unsafe {
let head = *KERNEL.ring_head.get();
if !head.is_null() {
let mut p = head;
loop {
if (*p).state == TaskState::Blocked && (*p).blocked_on == resource {
(*p).flags |= crate::tcb::TCB_FLAG_WOKEN;
(*p).state = TaskState::Ready;
(*p).blocked_on = 0;
(*p).block_deadline = 0;
found = true;
break;
}
p = (*p).next;
if p.is_null() || p == head {
break;
}
}
}
}
drop(g);
found
}
pub fn blocked_on_count(resource: u32) -> usize {
let g = critical::enter();
let mut n = 0usize;
unsafe {
let head = *KERNEL.ring_head.get();
if !head.is_null() {
let mut p = head;
loop {
if (*p).state == TaskState::Blocked && (*p).blocked_on == resource {
n += 1;
}
p = (*p).next;
if p.is_null() || p == head {
break;
}
}
}
}
drop(g);
n
}
pub fn yield_now() {
crate::arch::request_switch();
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TaskId(u32);
impl TaskId {
pub const fn from_raw(raw: u32) -> TaskId {
TaskId(raw)
}
pub const fn raw(self) -> u32 {
self.0
}
pub const fn is_none(self) -> bool {
self.0 == 0
}
}
pub fn current_id() -> TaskId {
TaskId(current_task_id())
}
struct ResourceIds(UnsafeCell<u32>);
unsafe impl Sync for ResourceIds {}
static NEXT_RESOURCE_ID: ResourceIds = ResourceIds(UnsafeCell::new(0x4000_0000));
pub fn alloc_resource_id() -> u32 {
let g = critical::enter();
let id = unsafe { *NEXT_RESOURCE_ID.0.get() };
unsafe { *NEXT_RESOURCE_ID.0.get() = id.wrapping_add(1) };
drop(g);
id
}
pub fn block_for_ticks(resource: u32, ticks: u64) -> BlockOutcome {
let g = critical::enter();
let deadline = unsafe { *KERNEL.ticks.get() }.wrapping_add(ticks);
let outcome = unsafe { mark_blocked_locked(resource, deadline) };
drop(g);
outcome
}
pub fn deadline_after_ticks(ticks: u64) -> u64 {
let g = critical::enter();
let d = unsafe { *KERNEL.ticks.get() }.wrapping_add(ticks);
drop(g);
d
}
pub fn sleep_ticks(ticks: u64) {
let _ = block_for_ticks(0, ticks);
}
pub fn sleep_until_tick(deadline_tick: u64) {
while tick_count() < deadline_tick {
match block_until_tick(0, deadline_tick) {
BlockOutcome::Blocked => {}
BlockOutcome::Ready => {}
BlockOutcome::AlreadyPast => break,
BlockOutcome::NoCurrentTask => break,
}
}
}
pub fn wake_blocked_on(resource: u32) {
let g = critical::enter();
unsafe {
let head = *KERNEL.ring_head.get();
if !head.is_null() {
let mut p = head;
loop {
if (*p).state == TaskState::Blocked && (*p).blocked_on == resource {
(*p).state = TaskState::Ready;
(*p).blocked_on = 0;
(*p).block_deadline = 0;
}
p = (*p).next;
if p.is_null() || p == head {
break;
}
}
}
}
drop(g);
}
unsafe fn wake_expired() {
let now = *KERNEL.ticks.get();
let head = *KERNEL.ring_head.get();
if head.is_null() {
return;
}
let mut p = head;
loop {
if (*p).state == TaskState::Blocked
&& (*p).block_deadline != 0
&& now >= (*p).block_deadline
{
(*p).state = TaskState::Ready;
(*p).blocked_on = 0;
(*p).block_deadline = 0;
}
p = (*p).next;
if p.is_null() || p == head {
break;
}
}
}
pub fn blocks_without_current() -> u64 {
let g = critical::enter();
let n = unsafe { *KERNEL.blocks_without_current.get() };
drop(g);
n
}
pub fn blocked_threads() -> usize {
let g = critical::enter();
let mut n = 0usize;
unsafe {
let head = *KERNEL.ring_head.get();
if !head.is_null() {
let mut p = head;
loop {
if (*p).state == TaskState::Blocked {
n += 1;
}
p = (*p).next;
if p.is_null() || p == head {
break;
}
}
}
}
drop(g);
n
}
pub fn set_active_cores(n: usize) -> Result<(), ConfigError> {
let max = crate::smp::max_cores().max(1);
if n == 0 || n > max {
return Err(ConfigError::Platform(
"active_cores outside 1..=smp::max_cores() for this target",
));
}
if n > 1 && !crate::smp::supports_smp() {
return Err(ConfigError::Platform(
"this target has no cross-core interlock, so it cannot run more than one core",
));
}
let g = critical::enter();
unsafe { (*KERNEL.config.get()).active_cores = n };
drop(g);
Ok(())
}
pub fn active_cores() -> usize {
let g = critical::enter();
let n = unsafe { (*KERNEL.config.get()).active_cores };
drop(g);
n
}
pub(crate) unsafe fn alloc_tcb() -> Option<*mut TaskControlBlock> {
let p = (*arena()).alloc(
core::mem::size_of::<TaskControlBlock>(),
core::mem::align_of::<TaskControlBlock>(),
)? as *mut TaskControlBlock;
ptr::write_bytes(p as *mut u8, 0, core::mem::size_of::<TaskControlBlock>());
Some(p)
}
pub(crate) unsafe fn take_id() -> u32 {
let id = *KERNEL.next_id.get();
*KERNEL.next_id.get() = id.wrapping_add(1);
id
}
pub unsafe fn reclaim_finished_tasks() {
let mut node = *KERNEL.pending_free.get();
*KERNEL.pending_free.get() = ptr::null_mut();
let ar = &mut *arena();
while !node.is_null() {
let next = (*node).prev;
debug_assert!(!(*node).is_linked(), "dead task still linked in the ring");
debug_assert_eq!((*node).state, TaskState::Dead);
if crate::arch::is_pinned(node) {
(*node).prev = *KERNEL.pending_free.get();
*KERNEL.pending_free.get() = node;
node = next;
continue;
}
crate::arch::on_task_reclaimed(node);
ar.free((*node).closure_block);
ar.free((*node).stack_base);
let n = node;
node = next;
ar.free(n as *mut u8);
*KERNEL.reclaimed.get() += 1;
}
}
pub(crate) unsafe fn push_pending_free(tcb: *mut TaskControlBlock) {
(*tcb).prev = *KERNEL.pending_free.get();
*KERNEL.pending_free.get() = tcb;
}
pub(crate) unsafe fn spawn_internal<F>(f: F, stack_size: usize) -> Result<(), crate::SpawnError>
where
F: FnOnce() + Send + 'static,
{
let g = critical::enter();
let ar = arena();
if !(*ar).is_ready() {
drop(g);
return Err(crate::SpawnError::NotInitialized);
}
let tcb = match alloc_tcb() {
Some(t) => t,
None => {
drop(g);
return Err(crate::SpawnError::ArenaExhausted);
}
};
let csize = crate::closure::block_size_for::<F>();
let calign = crate::closure::block_align_for::<F>();
let blob = match (*arena()).alloc(csize, calign) {
Some(b) => b,
None => {
(*arena()).free(tcb as *mut u8);
drop(g);
return Err(crate::SpawnError::ArenaExhausted);
}
};
crate::closure::place::<F>(blob, f);
(*tcb).closure_block = blob;
(*tcb).id = take_id();
(*tcb).state = TaskState::Ready;
(*tcb).slices_run = 0;
(*tcb).switches = 0;
if let Err(e) = crate::arch::create_task(tcb, stack_size) {
crate::closure::drop_blob::<F>(blob);
(*arena()).free(blob);
(*arena()).free(tcb as *mut u8);
drop(g);
return Err(e);
}
let cur = KERNEL.current();
crate::ring::insert_after(cur, tcb);
if cur.is_null() {
(*tcb).state = TaskState::Running;
KERNEL.set_current(tcb);
}
*KERNEL.ring_head.get() = tcb;
*KERNEL.total_threads.get() += 1;
*KERNEL.active_threads.get() += 1;
debug_assert!(crate::ring::check(KERNEL.current()).is_ok());
if *KERNEL.running.get() {
crate::arch::request_switch();
}
drop(g);
Ok(())
}