use crate::config::{ConfigError, PlatformLimits, SchedulerConfig, Slice};
use crate::scheduler;
use crate::tcb::{IdlePolicy, TaskControlBlock, TaskState, KERNEL};
use core::ffi::c_void;
use core::ptr;
use core::sync::atomic::{AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering};
use std::sync::Once;
pub type Handle = *mut c_void;
type Bool = i32;
type Dword = u32;
type Lpvoid = *mut c_void;
type ThreadStart = unsafe extern "system" fn(Lpvoid) -> Dword;
#[link(name = "kernel32")]
unsafe extern "system" {
fn CreateThread(
attrs: Lpvoid,
stack_size: usize,
start: ThreadStart,
param: Lpvoid,
flags: Dword,
thread_id: *mut Dword,
) -> Handle;
fn ResumeThread(thread: Handle) -> Dword;
fn SuspendThread(thread: Handle) -> Dword;
fn OpenThread(access: Dword, inherit: Bool, thread_id: Dword) -> Handle;
fn GetCurrentThreadId() -> Dword;
fn CloseHandle(handle: Handle) -> Bool;
fn CreateEventW(attrs: Lpvoid, manual_reset: Bool, initial: Bool, name: *const u16) -> Handle;
fn SetEvent(handle: Handle) -> Bool;
fn WaitForMultipleObjects(
count: Dword,
handles: *const Handle,
wait_all: Bool,
timeout_ms: Dword,
) -> Dword;
fn CreateWaitableTimerExW(
attrs: Lpvoid,
name: *const u16,
flags: Dword,
access: Dword,
) -> Handle;
fn CancelWaitableTimer(timer: Handle) -> Bool;
fn SetWaitableTimer(
timer: Handle,
due_time: *const i64,
period_ms: i32,
completion: Lpvoid,
arg: Lpvoid,
resume: Bool,
) -> Bool;
fn SwitchToThread() -> Bool;
fn Sleep(ms: Dword);
fn SetThreadPriority(thread: Handle, priority: i32) -> Bool;
fn ExitThread(code: Dword) -> !;
fn GetCurrentProcess() -> Handle;
fn TerminateProcess(process: Handle, code: Dword) -> Bool;
fn QueryPerformanceCounter(out: *mut i64) -> Bool;
fn QueryPerformanceFrequency(out: *mut i64) -> Bool;
}
#[link(name = "winmm")]
unsafe extern "system" {
fn timeBeginPeriod(period: u32) -> u32;
fn timeEndPeriod(period: u32) -> u32;
}
const CREATE_SUSPENDED: Dword = 0x0000_0004;
const CREATE_WAITABLE_TIMER_HIGH_RESOLUTION: Dword = 0x0000_0002;
const TIMER_ALL_ACCESS: Dword = 0x001F_0003;
const THREAD_SUSPEND_RESUME: Dword = 0x0002;
const THREAD_TERMINATE: Dword = 0x0001;
const THREAD_QUERY_INFORMATION: Dword = 0x0040;
const THREAD_PRIORITY_TIME_CRITICAL: i32 = 15;
const INFINITE: Dword = 0xFFFF_FFFF;
const WAIT_OBJECT_0: Dword = 0;
const TIMER_UNITS_PER_SEC: u32 = 10_000_000;
const MIN_SLICE_NS_HIGH_RES: u64 = 500_000;
const MIN_SLICE_NS_FALLBACK: u64 = 1_000_000;
static LOCK_OWNER: AtomicU32 = AtomicU32::new(0);
static LOCK_DEPTH: AtomicU32 = AtomicU32::new(0);
pub type CriticalToken = u32;
pub unsafe fn critical_enter() -> CriticalToken {
let me = GetCurrentThreadId().wrapping_add(1);
if LOCK_OWNER.load(Ordering::Acquire) == me {
return LOCK_DEPTH.fetch_add(1, Ordering::Relaxed).wrapping_add(1);
}
let mut spins: u64 = 0;
loop {
if LOCK_OWNER
.compare_exchange_weak(0, me, Ordering::Acquire, Ordering::Relaxed)
.is_ok()
{
LOCK_DEPTH.store(1, Ordering::Relaxed);
return 1;
}
spins += 1;
if spins & 0xFF == 0 {
SwitchToThread();
}
if spins > 200_000_000 {
std::process::abort();
}
core::hint::spin_loop();
}
}
pub unsafe fn critical_exit(_token: CriticalToken) {
if LOCK_DEPTH.fetch_sub(1, Ordering::AcqRel) == 1 {
LOCK_OWNER.store(0, Ordering::Release);
}
}
static TIMER: AtomicUsize = AtomicUsize::new(0);
static KICK: AtomicUsize = AtomicUsize::new(0);
static TICK_THREAD: AtomicUsize = AtomicUsize::new(0);
static EXIT_CODE: AtomicU32 = AtomicU32::new(0);
static RUNNING_TCB: AtomicUsize = AtomicUsize::new(0);
static SLICE_UNITS: AtomicU64 = AtomicU64::new(10_000);
static DEADLINE: AtomicU64 = AtomicU64::new(0);
static LAST_TICK_NS: AtomicU64 = AtomicU64::new(0);
static HIGH_RES: AtomicU8 = AtomicU8::new(0);
static OBJECTS: Once = Once::new();
unsafe fn ensure_objects() {
OBJECTS.call_once(|| {
timeBeginPeriod(1);
let mut timer = CreateWaitableTimerExW(
ptr::null_mut(),
ptr::null(),
CREATE_WAITABLE_TIMER_HIGH_RESOLUTION,
TIMER_ALL_ACCESS,
);
if timer.is_null() {
HIGH_RES.store(2, Ordering::Release);
timer = CreateWaitableTimerExW(ptr::null_mut(), ptr::null(), 0, TIMER_ALL_ACCESS);
} else {
HIGH_RES.store(1, Ordering::Release);
}
TIMER.store(timer as usize, Ordering::Release);
let kick = CreateEventW(ptr::null_mut(), 0, 0, ptr::null());
KICK.store(kick as usize, Ordering::Release);
});
}
unsafe fn rearm_timer() {
let units = SLICE_UNITS.load(Ordering::Relaxed) as i64;
let due: i64 = -units; DEADLINE.store(now_ns() + (units as u64) * 100, Ordering::Relaxed);
SetWaitableTimer(
TIMER.load(Ordering::Relaxed) as Handle,
&due,
0,
ptr::null_mut(),
ptr::null_mut(),
0,
);
}
static QPC_FREQ: AtomicU64 = AtomicU64::new(0);
unsafe fn now_ns() -> u64 {
let mut freq = QPC_FREQ.load(Ordering::Relaxed);
if freq == 0 {
let mut f: i64 = 0;
if QueryPerformanceFrequency(&mut f) == 0 || f <= 0 {
return 0; }
freq = f as u64;
QPC_FREQ.store(freq, Ordering::Relaxed);
}
let mut c: i64 = 0;
if QueryPerformanceCounter(&mut c) == 0 {
return 0;
}
((c as u128) * 1_000_000_000u128 / freq as u128) as u64
}
unsafe fn suspend_and_maybe_close(prev: *mut TaskControlBlock, _next: Handle) {
let h = (*prev).backend as Handle;
if h.is_null() {
return;
}
SuspendThread(h);
if (*prev).state == TaskState::Dead {
CloseHandle(h);
(*prev).backend = ptr::null_mut();
}
}
pub fn exit_process(code: i32) -> ! {
use std::io::Write;
let _ = std::io::stdout().flush();
let _ = std::io::stderr().flush();
unsafe {
CancelWaitableTimer(TIMER.load(Ordering::Relaxed) as Handle);
TerminateProcess(GetCurrentProcess(), code as Dword);
}
loop {
core::hint::spin_loop();
}
}
unsafe fn finish_shutdown(code: u32) -> ! {
let mut start = *KERNEL.ring_head.get();
if start.is_null() {
start = KERNEL.current();
}
if !start.is_null() {
let mut p = start;
loop {
let h = (*p).backend as Handle;
if !h.is_null() {
ResumeThread(h);
}
p = (*p).next;
if p.is_null() || p == start {
break;
}
}
}
timeEndPeriod(1);
exit_process(code as i32)
}
unsafe extern "system" fn tick_thread_main(_param: Lpvoid) -> Dword {
SetThreadPriority(
OpenThread(THREAD_SUSPEND_RESUME, 0, GetCurrentThreadId()),
THREAD_PRIORITY_TIME_CRITICAL,
);
loop {
let handles = [
TIMER.load(Ordering::Relaxed) as Handle,
KICK.load(Ordering::Relaxed) as Handle,
];
let woke = WaitForMultipleObjects(2, handles.as_ptr(), 0, INFINITE);
let from_timer = woke == WAIT_OBJECT_0;
let t0 = now_ns();
if from_timer {
let last = LAST_TICK_NS.swap(t0, Ordering::Relaxed);
if last != 0 && t0 > last {
let period = t0 - last;
let want = SLICE_UNITS.load(Ordering::Relaxed) * 100;
let err = period.abs_diff(want);
scheduler::record_period_error(err.min(u32::MAX as u64) as u32);
}
}
let token = critical_enter();
if *KERNEL.shutdown.get() {
finish_shutdown(EXIT_CODE.load(Ordering::Relaxed));
}
if from_timer {
scheduler::on_tick();
}
scheduler::reclaim_finished_tasks();
let next = scheduler::schedule_next();
if next.is_null() {
let policy = (*KERNEL.config.get()).idle;
if policy == IdlePolicy::ExitWhenAllDead {
finish_shutdown(0);
}
let prev_ptr = RUNNING_TCB.swap(0, Ordering::AcqRel);
if prev_ptr != 0 {
suspend_and_maybe_close(prev_ptr as *mut TaskControlBlock, ptr::null_mut());
}
rearm_timer();
} else {
let next_handle = (*next).backend as Handle;
let prev_ptr = RUNNING_TCB.swap(next as usize, Ordering::AcqRel);
if prev_ptr != 0 && prev_ptr != next as usize {
suspend_and_maybe_close(prev_ptr as *mut TaskControlBlock, next_handle);
}
ResumeThread(next_handle);
rearm_timer();
}
critical_exit(token);
let dt = now_ns().saturating_sub(t0);
scheduler::record_latency(dt.min(u32::MAX as u64) as u32);
}
}
use crate::arch::TimerPlan;
pub fn cycles_to_ns(cycles: u32) -> u64 {
cycles as u64 * (1_000_000_000 / TIMER_UNITS_PER_SEC as u64)
}
pub fn platform_limits() -> PlatformLimits {
unsafe { ensure_objects() };
let high_res = HIGH_RES.load(Ordering::Acquire) == 1;
PlatformLimits {
min_slice_ns: if high_res {
MIN_SLICE_NS_HIGH_RES
} else {
MIN_SLICE_NS_FALLBACK
},
max_slice_ns: Some(u32::MAX as u64 * 100),
timer_hz: TIMER_UNITS_PER_SEC,
timer_note: if high_res {
"high-resolution waitable timer (0.5 ms granularity)"
} else {
"system waitable timer with timeBeginPeriod(1) (1 ms granularity)"
},
}
}
pub fn plan_timer(cfg: &SchedulerConfig) -> Result<TimerPlan, ConfigError> {
let limits = platform_limits();
let requested = match cfg.slice {
Slice::Cycles(c) => c as u64 * 100,
other => other.as_nanos(cfg.timer_hz),
};
if requested == 0 {
return Err(ConfigError::ZeroSlice);
}
if requested < limits.min_slice_ns {
return Err(ConfigError::SliceBelowPlatformMinimum {
requested_ns: requested,
minimum_ns: limits.min_slice_ns,
});
}
let units = requested.div_ceil(100);
if units > u32::MAX as u64 {
return Err(ConfigError::SliceAboveTimerRange {
requested_ns: requested,
maximum_ns: u32::MAX as u64 * 100,
});
}
Ok(TimerPlan {
slice_ns: units * 100,
slice_cycles: units as u32,
timer_hz: TIMER_UNITS_PER_SEC,
})
}
pub fn start_timer(plan: TimerPlan) -> Result<(), ConfigError> {
unsafe {
ensure_objects();
if TIMER.load(Ordering::Acquire) == 0 || KICK.load(Ordering::Acquire) == 0 {
return Err(ConfigError::Platform("could not create the kernel timer"));
}
SLICE_UNITS.store(plan.slice_cycles as u64, Ordering::Release);
rearm_timer();
let handle = CreateThread(
ptr::null_mut(),
64 * 1024,
tick_thread_main,
ptr::null_mut(),
0,
ptr::null_mut(),
);
if handle.is_null() {
return Err(ConfigError::Platform("could not create the tick thread"));
}
TICK_THREAD.store(handle as usize, Ordering::Release);
SetThreadPriority(handle, THREAD_PRIORITY_TIME_CRITICAL);
}
Ok(())
}
pub fn retune_timer(plan: TimerPlan) -> Result<(), ConfigError> {
unsafe {
SLICE_UNITS.store(plan.slice_cycles as u64, Ordering::Release);
rearm_timer();
}
Ok(())
}
pub unsafe fn adopt_current_task(tcb: *mut TaskControlBlock) -> Result<(), ConfigError> {
unsafe {
let handle = OpenThread(
THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION | THREAD_TERMINATE,
0,
GetCurrentThreadId(),
);
if handle.is_null() {
return Err(ConfigError::Platform(
"OpenThread on the calling thread failed",
));
}
(*tcb).backend = handle as *mut u8;
(*tcb).stack_base = ptr::null_mut();
(*tcb).stack_size = 0;
RUNNING_TCB.store(tcb as usize, Ordering::Release);
}
Ok(())
}
pub unsafe fn create_task(
tcb: *mut TaskControlBlock,
stack_size: usize,
) -> Result<(), crate::SpawnError> {
unsafe {
let size = stack_size.max(64 * 1024);
let mut tid: Dword = 0;
let handle = CreateThread(
ptr::null_mut(),
size,
task_thread_main,
tcb as Lpvoid,
CREATE_SUSPENDED,
&mut tid,
);
if handle.is_null() {
return Err(crate::SpawnError::Backend(ConfigError::Platform(
"CreateThread failed for a new task",
)));
}
(*tcb).backend = handle as *mut u8;
(*tcb).stack_base = ptr::null_mut();
(*tcb).stack_size = stack_size;
}
Ok(())
}
unsafe extern "system" fn task_thread_main(param: Lpvoid) -> Dword {
crate::trampoline::task_trampoline(param as *mut TaskControlBlock)
}
pub fn request_switch() {
unsafe {
let k = KICK.load(Ordering::Acquire);
if k != 0 {
SetEvent(k as Handle);
}
}
}
pub fn mark_running() {}
pub unsafe fn on_task_reclaimed(tcb: *mut TaskControlBlock) {
unsafe {
let h = (*tcb).backend as Handle;
(*tcb).backend = ptr::null_mut();
if h.is_null() {
return;
}
if h as usize != RUNNING_TCB.load(Ordering::Acquire) {
CloseHandle(h);
}
}
}
pub unsafe fn is_pinned(tcb: *mut TaskControlBlock) -> bool {
let v = RUNNING_TCB.load(Ordering::Acquire);
v != 0 && v == tcb as usize
}
pub fn exit_current_task_forever() -> ! {
unsafe { ExitThread(0) }
}
pub fn idle_forever() -> ! {
unsafe {
loop {
Sleep(1);
}
}
}
pub fn shutdown(code: i32) -> ! {
unsafe {
if TICK_THREAD.load(Ordering::Acquire) == 0 {
exit_process(code);
}
EXIT_CODE.store(code as u32, Ordering::Relaxed);
request_switch();
let spins = std::sync::atomic::AtomicU32::new(0);
loop {
Sleep(1);
if spins.fetch_add(1, Ordering::Relaxed) > 200 {
exit_process(code);
}
}
}
}
pub fn parks_synchronously() -> bool {
false
}
pub fn cycle_counter() -> Option<u32> {
None
}
pub fn cycle_counter_hz() -> u32 {
0
}