use crate::config::{ConfigError, PlatformLimits, SchedulerConfig, Slice};
use crate::tcb::{TaskControlBlock, KERNEL};
use core::ffi::{c_int, c_void};
use core::ptr;
use core::sync::atomic::{AtomicU32, AtomicU64, Ordering};
pub type SigSet = [u64; 16];
#[repr(C)]
pub struct SigAction {
pub sa_handler: usize,
pub sa_mask: SigSet,
pub sa_flags: c_int,
pub sa_restorer: usize,
}
#[repr(C)]
#[derive(Clone, Copy)]
pub struct TimeSpec {
pub tv_sec: i64,
pub tv_nsec: i64,
}
#[repr(C)]
#[derive(Clone, Copy)]
pub struct TimeVal {
pub tv_sec: i64,
pub tv_usec: i64,
}
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ITimerVal {
pub it_interval: TimeVal,
pub it_value: TimeVal,
}
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ITimerSpec {
pub it_interval: TimeSpec,
pub it_value: TimeSpec,
}
#[repr(C)]
#[derive(Clone, Copy)]
pub union SigVal {
pub sival_ptr: *mut c_void,
pub sival_int: c_int,
}
#[repr(C)]
#[derive(Clone, Copy)]
pub struct SigEvent {
pub sigev_value: SigVal,
pub sigev_signo: c_int,
pub sigev_notify: c_int,
pub sigev_un: [usize; 2],
}
pub type TimerId = *mut c_void;
unsafe extern "C" {
fn sigaction(signum: c_int, act: *const SigAction, old: *mut SigAction) -> c_int;
fn sigemptyset(set: *mut SigSet) -> c_int;
fn sigaddset(set: *mut SigSet, signum: c_int) -> c_int;
fn sigprocmask(how: c_int, set: *const SigSet, old: *mut SigSet) -> c_int;
fn raise(sig: c_int) -> c_int;
fn timer_create(clockid: c_int, evp: *mut SigEvent, timer: *mut TimerId) -> c_int;
fn timer_settime(
timer: TimerId,
flags: c_int,
new: *const ITimerSpec,
old: *mut ITimerSpec,
) -> c_int;
fn clock_gettime(clockid: c_int, tp: *mut TimeSpec) -> c_int;
}
const SIGALRM: c_int = 14;
const SIG_BLOCK: c_int = 0;
const SIG_UNBLOCK: c_int = 1;
const SIG_SETMASK: c_int = 2;
const SA_RESTART: c_int = 0x1000_0000;
const CLOCK_MONOTONIC: c_int = 1;
const SIGEV_SIGNAL: c_int = 0;
const MIN_SLICE_NS: u64 = 50_000;
const TIMER_UNITS_PER_SEC: u32 = 10_000_000;
const _: () = assert!(core::mem::size_of::<SigSet>() == 128);
const _: () = assert!(core::mem::size_of::<SigAction>() == 152);
const _: () = assert!(core::mem::size_of::<SigEvent>() == 32);
const _: () = assert!(core::mem::size_of::<ITimerSpec>() == 32);
const _: () = assert!(core::mem::align_of::<SigAction>() == 8);
pub type CriticalToken = SigSet;
pub unsafe fn critical_enter() -> CriticalToken {
let mut set: SigSet = [0; 16];
let mut old: SigSet = [0; 16];
sigemptyset(&mut set);
sigaddset(&mut set, SIGALRM);
sigprocmask(SIG_BLOCK, &set, &mut old);
old
}
pub unsafe fn critical_exit(token: CriticalToken) {
sigprocmask(SIG_SETMASK, &token, ptr::null_mut());
}
core::arch::global_asm!(
r#"
.text
/* void rrkernel_switch(void **from_sp, void *to_sp)
* rdi = where to store our SP, rsi = the SP to resume.
*
* Saved frame, ascending from the stored SP:
* [ 0..16) MXCSR (4) , x87 control word (2), padding
* [16..64) r15, r14, r13, r12, rbx, rbp
* [64..72) return address
* SysV callee-saved state is exactly rbx/rbp/r12-r15 plus the two control
* registers; anything else is caller-saved and already dead at a call site.
*/
.globl rrkernel_switch
.type rrkernel_switch, @function
rrkernel_switch:
push rbp
push rbx
push r12
push r13
push r14
push r15
sub rsp, 16
stmxcsr [rsp]
fnstcw [rsp+4]
mov [rdi], rsp
mov rsp, rsi
ldmxcsr [rsp]
fldcw [rsp+4]
add rsp, 16
pop r15
pop r14
pop r13
pop r12
pop rbx
pop rbp
ret
/* A new fibre lands here after its first switch: r12 holds the TCB pointer
* that create_task planted in the frame.
*
* First it unblocks the tick signal: this fibre was born inside a signal
* handler's callee chain (that is where the switch happened), so the
* thread's SIGALRM mask is still "blocked" and no tick would ever reach it
* again. Then `jmp` (not `call`) into the trampoline keeps the stack
* alignment the ABI expects at a function entry; the trampoline never
* returns. */
.globl rrkernel_task_start
.type rrkernel_task_start, @function
rrkernel_task_start:
sub rsp, 8
call rrkernel_fibre_entry
add rsp, 8
mov rdi, r12
jmp task_trampoline
"#
);
unsafe extern "C" {
fn rrkernel_switch(from_sp: *mut *mut u8, to_sp: *mut u8);
}
static TIMER: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
static SLICE_NS: AtomicU64 = AtomicU64::new(1_000_000);
static LAST_TICK_NS: AtomicU64 = AtomicU64::new(0);
static NEXT_EXPIRY_NS: AtomicU64 = AtomicU64::new(0);
static READY: AtomicU32 = AtomicU32::new(0);
fn now_ns() -> u64 {
let mut ts = TimeSpec {
tv_sec: 0,
tv_nsec: 0,
};
unsafe {
if clock_gettime(CLOCK_MONOTONIC, &mut ts) != 0 {
return 0;
}
}
(ts.tv_sec as u64)
.saturating_mul(1_000_000_000)
.saturating_add(ts.tv_nsec as u64)
}
unsafe fn rearm() {
let ns = SLICE_NS.load(Ordering::Relaxed);
NEXT_EXPIRY_NS.store(now_ns().saturating_add(ns), Ordering::Relaxed);
let spec = ITimerSpec {
it_interval: TimeSpec {
tv_sec: 0,
tv_nsec: 0,
},
it_value: TimeSpec {
tv_sec: (ns / 1_000_000_000) as i64,
tv_nsec: (ns % 1_000_000_000) as i64,
},
};
timer_settime(
TIMER.load(Ordering::Relaxed) as TimerId,
0,
&spec,
ptr::null_mut(),
);
}
#[no_mangle]
extern "C" fn rrkernel_fibre_entry() {
unsafe {
let mut set: SigSet = [0; 16];
sigemptyset(&mut set);
sigaddset(&mut set, SIGALRM);
sigprocmask(SIG_UNBLOCK, &set, ptr::null_mut());
}
}
static SCRATCH_SP: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
extern "C" fn alarm_handler(_sig: c_int) {
unsafe {
let t0 = now_ns();
let from_timer = t0 >= NEXT_EXPIRY_NS.load(Ordering::Relaxed);
if from_timer {
crate::scheduler::on_tick();
let cfg = *KERNEL.config.get();
if cfg.measure != crate::tcb::Measure::Off {
let last = LAST_TICK_NS.swap(t0, Ordering::Relaxed);
if last != 0 && t0 > last {
let want = SLICE_NS.load(Ordering::Relaxed);
let err = (t0 - last).abs_diff(want);
crate::scheduler::record_period_error(err.min(u32::MAX as u64) as u32);
}
}
}
let cur = KERNEL.current();
let next = crate::scheduler::schedule_next();
if next.is_null() || next == cur {
rearm();
return;
}
rearm();
if (*KERNEL.config.get()).measure != crate::tcb::Measure::Off {
let dt = now_ns().saturating_sub(t0);
crate::scheduler::record_latency(dt.min(u32::MAX as u64) as u32);
}
let from: *mut *mut u8 = if cur.is_null() {
SCRATCH_SP.as_ptr() as *mut *mut u8
} else {
&mut (*cur).sp as *mut *mut u8
};
rrkernel_switch(from, (*next).sp);
crate::scheduler::reclaim_finished_tasks();
}
}
pub fn cycles_to_ns(cycles: u32) -> u64 {
cycles as u64 * (1_000_000_000 / TIMER_UNITS_PER_SEC as u64)
}
pub fn platform_limits() -> PlatformLimits {
PlatformLimits {
min_slice_ns: MIN_SLICE_NS,
max_slice_ns: Some(u32::MAX as u64 * 100),
timer_hz: TIMER_UNITS_PER_SEC,
timer_note: "POSIX hrtimer (CLOCK_MONOTONIC) + SIGALRM, x86-64 fibre switch",
}
}
pub fn plan_timer(cfg: &SchedulerConfig) -> Result<crate::arch::TimerPlan, ConfigError> {
let requested = match cfg.slice {
Slice::Cycles(c) => c as u64 * 100,
other => other.as_nanos(cfg.timer_hz),
};
if requested == 0 {
return Err(ConfigError::ZeroSlice);
}
if requested < MIN_SLICE_NS {
return Err(ConfigError::SliceBelowPlatformMinimum {
requested_ns: requested,
minimum_ns: MIN_SLICE_NS,
});
}
let units = requested.div_ceil(100);
if units > u32::MAX as u64 {
return Err(ConfigError::SliceAboveTimerRange {
requested_ns: requested,
maximum_ns: u32::MAX as u64 * 100,
});
}
Ok(crate::arch::TimerPlan {
slice_ns: units * 100,
slice_cycles: units as u32,
timer_hz: TIMER_UNITS_PER_SEC,
})
}
pub fn start_timer(plan: crate::arch::TimerPlan) -> Result<(), ConfigError> {
unsafe {
SLICE_NS.store(plan.slice_ns, Ordering::Release);
let mut act = SigAction {
sa_handler: alarm_handler as *const () as usize,
sa_mask: [0; 16],
sa_flags: SA_RESTART,
sa_restorer: 0,
};
sigemptyset(&mut act.sa_mask);
if sigaction(SIGALRM, &act, ptr::null_mut()) != 0 {
return Err(ConfigError::Platform("sigaction(SIGALRM) failed"));
}
let mut ev = SigEvent {
sigev_value: SigVal { sival_int: 0 },
sigev_signo: SIGALRM,
sigev_notify: SIGEV_SIGNAL,
sigev_un: [0; 2],
};
let mut id: TimerId = ptr::null_mut();
if timer_create(CLOCK_MONOTONIC, &mut ev, &mut id) != 0 {
return Err(ConfigError::Platform("timer_create failed"));
}
TIMER.store(id as usize, Ordering::Release);
rearm();
READY.store(1, Ordering::Release);
}
Ok(())
}
pub fn retune_timer(plan: crate::arch::TimerPlan) -> Result<(), ConfigError> {
unsafe {
SLICE_NS.store(plan.slice_ns, Ordering::Release);
rearm();
}
Ok(())
}
pub unsafe fn adopt_current_task(tcb: *mut TaskControlBlock) -> Result<(), ConfigError> {
unsafe {
(*tcb).sp = ptr::null_mut();
(*tcb).stack_base = ptr::null_mut();
(*tcb).stack_size = 0;
}
Ok(())
}
#[inline]
fn rrkernel_task_start_addr() -> usize {
unsafe extern "C" {
fn rrkernel_task_start();
}
rrkernel_task_start as *const () as usize
}
pub unsafe fn create_task(
tcb: *mut TaskControlBlock,
stack_size: usize,
) -> Result<(), crate::SpawnError> {
let size = stack_size.max(8 * 1024) + 16;
let block = {
let guard = crate::critical::enter();
let p = unsafe { (*crate::scheduler::arena()).alloc(size, 16) };
drop(guard);
p
};
let block = match block {
Some(p) => p,
None => return Err(crate::SpawnError::ArenaExhausted),
};
unsafe {
(*tcb).stack_base = block;
(*tcb).stack_size = size - 16;
let top = (block as usize + size) & !15;
let frame = ((top - 72) & !15) as *mut u8;
ptr::write_bytes(frame, 0, 72);
ptr::write_volatile(frame as *mut u32, 0x0000_1F80); ptr::write_volatile(frame.add(4) as *mut u16, 0x037F); ptr::write_volatile(frame.add(40) as *mut usize, tcb as usize);
ptr::write_volatile(frame.add(64) as *mut usize, rrkernel_task_start_addr());
(*tcb).sp = frame;
(*tcb).flags = 0;
}
Ok(())
}
pub fn mark_running() {}
pub fn exit_current_task_forever() -> ! {
loop {
core::hint::spin_loop();
}
}
pub fn idle_forever() -> ! {
loop {
core::hint::spin_loop();
}
}
pub fn request_switch() {
unsafe {
if READY.load(Ordering::Acquire) == 1 {
raise(SIGALRM);
}
}
}
pub fn shutdown(code: i32) -> ! {
std::process::exit(code)
}
pub unsafe fn on_task_reclaimed(_tcb: *mut TaskControlBlock) {}
pub unsafe fn is_pinned(_tcb: *mut TaskControlBlock) -> bool {
false
}
pub fn parks_synchronously() -> bool {
false
}
pub fn cycle_counter() -> Option<u32> {
None
}
pub fn cycle_counter_hz() -> u32 {
0
}