use super::Kernel;
use crate::{CarrierStatus, TaskFailure, TaskStatus, kernel_tasks::OwnedTask};
#[cfg(feature = "handoff-profiling")]
use crate::{
handoff_profile::HandoffStage,
handoff_span::{Span, duration},
};
use std::sync::Arc;
use vthread_stack::Fiber;
const REMOTE_READY_TARGET: usize = 64;
const REMOTE_ADMISSION_DISPATCH_BOUND: u32 = 65_536;
const IDLE_SIGNAL_PROBES: usize = 640;
const SPINS_PER_SIGNAL_PROBE: usize = 1;
impl Kernel {
pub(crate) fn receive(&mut self) -> bool {
#[cfg(test)]
self.record_test_progress(crate::signal::TestCarrierPhase::Receive);
self.admission_pressure += u32::from(self.remote_pending);
let received = self.receive_local_tasks();
let received = self.receive_remote_tasks() || received;
if received {
self.publish(CarrierStatus::Running);
}
self.remote_pending = self.inbox.pending() != 0;
if !self.remote_pending {
self.admission_pressure = 0;
}
self.remote_pending
}
#[cfg(test)]
pub(crate) fn remote_pending(&self) -> bool {
self.remote_pending
}
pub(crate) fn remote_receive_required(&self) -> bool {
self.remote_pending || self.inbox.pending() != 0
}
pub(crate) fn receive_local(&mut self) {
if self.receive_local_tasks() {
self.publish(CarrierStatus::Running);
}
}
fn receive_local_tasks(&mut self) -> bool {
let mut received = false;
while let Some(task) = self.local.pop_start() {
received = true;
self.has_borrowed = true;
#[cfg(feature = "runtime-evidence")]
self.shared
.record(crate::diagnostics::evidence::RuntimeEventKind::QueueDepth {
carrier: self.id,
queue: crate::diagnostics::evidence::QueueKind::LocalStart,
depth: self.local.pending_starts(),
capacity: self.shared.config.carrier_queue_capacity(),
});
self.shared.transition(
task.execution.as_ref().expect("task execution").record(),
|record| record.status = TaskStatus::Ready,
);
self.ready.push_back(self.tasks.insert_borrowed(task));
}
received
}
fn receive_remote_tasks(&mut self) -> bool {
assert!(self.incoming.is_empty(), "unprocessed start batch");
let capacity = self.shared.config.carrier_queue_capacity();
let target = REMOTE_READY_TARGET.min(capacity);
let limit = if self.ready.len() <= target / 2 {
target - self.ready.len()
} else if self.admission_pressure >= REMOTE_ADMISSION_DISPATCH_BOUND {
1
} else {
#[cfg(feature = "scheduler-profiling")]
self.scheduler_profile.record_receive(None);
return false;
};
let drained = self.inbox.drain_into(&mut self.incoming, limit);
#[cfg(feature = "scheduler-profiling")]
self.scheduler_profile.record_receive(Some(drained));
if drained != 0 {
self.admission_pressure = 0;
}
let mut received = false;
while let Some(packet) = self.incoming.pop_front() {
self.pending = Some(packet);
#[cfg(feature = "lifecycle-profiling")]
let stack_fiber_started = std::time::Instant::now();
#[cfg(feature = "runtime-evidence")]
let acquired = self.local.stacks.borrow_mut().acquire_identified();
#[cfg(not(feature = "runtime-evidence"))]
let acquired = self.local.stacks.borrow_mut().acquire();
#[cfg(feature = "runtime-evidence")]
let (stack_identity, stack) = match acquired {
Ok(stack) => stack,
Err(_) => {
self.discard_pending(TaskFailure::StackAllocation);
#[cfg(feature = "lifecycle-profiling")]
self.shared
.lifecycle_probe
.record_stack_fiber(stack_fiber_started.elapsed());
continue;
}
};
#[cfg(not(feature = "runtime-evidence"))]
let stack = match acquired {
Ok(stack) => stack,
Err(_) => {
self.discard_pending(TaskFailure::StackAllocation);
#[cfg(feature = "lifecycle-profiling")]
self.shared
.lifecycle_probe
.record_stack_fiber(stack_fiber_started.elapsed());
continue;
}
};
let packet = self.pending.as_mut().expect("pending packet");
received = true;
let entry = packet.entry.take().expect("unstarted packet entry");
let body_record = Arc::clone(&packet.record);
let fiber = Fiber::new(stack, move || entry.run(&body_record));
#[cfg(feature = "runtime-evidence")]
let task_fiber = crate::task_fiber::OwnedFiber::new(fiber, stack_identity);
#[cfg(not(feature = "runtime-evidence"))]
let task_fiber = crate::task_fiber::OwnedFiber::new(fiber);
let (id, scope, options) = self.shared.transition(&packet.record, |record| {
record.status = TaskStatus::Ready;
(record.id, record.scope, record.options().clone())
});
let record = Arc::clone(&packet.record);
let execution = self.acquire_execution(id, scope, record, options);
let task = OwnedTask {
fiber: Some(task_fiber),
execution: Some(execution),
};
#[cfg(feature = "runtime-evidence")]
self.shared.record(
crate::diagnostics::evidence::RuntimeEventKind::StackCheckedOut {
task: id,
stack: crate::diagnostics::evidence::StackId::new(self.id, stack_identity),
},
);
self.pending = None;
self.ready.push_back(self.tasks.insert_owned(task));
#[cfg(feature = "lifecycle-profiling")]
self.shared
.lifecycle_probe
.record_stack_fiber(stack_fiber_started.elapsed());
}
received
}
pub(crate) fn wait_for_work(&mut self, observed: u64) {
#[cfg(test)]
self.record_test_progress(crate::signal::TestCarrierPhase::Idle);
#[cfg(feature = "handoff-profiling")]
let _episode = Span::new(HandoffStage::IdleEpisode);
#[cfg(feature = "scheduler-profiling")]
self.scheduler_profile.record_idle(self.stats.mounts);
self.flush_completions();
let deadline = self.timers.next_deadline();
if deadline.is_some() {
self.stats.timer_sleeps += 1;
}
if self.observe_idle_work() {
#[cfg(feature = "scheduler-profiling")]
self.scheduler_profile.record_early_work();
return;
}
if deadline.is_none() && self.shared.config.carriers() > 1 {
#[cfg(feature = "handoff-profiling")]
let polling = std::time::Instant::now();
for _probe in 0..IDLE_SIGNAL_PROBES {
for _ in 0..SPINS_PER_SIGNAL_PROBE {
std::hint::spin_loop();
}
let work = self.observe_idle_work();
if work || self.inbox.signal.version() != observed {
#[cfg(feature = "handoff-profiling")]
duration(
if work {
HandoffStage::PollWork
} else {
HandoffStage::PollControl
},
polling.elapsed(),
);
#[cfg(feature = "scheduler-profiling")]
self.scheduler_profile.record_poll(_probe + 1, true);
return;
}
}
#[cfg(feature = "handoff-profiling")]
duration(HandoffStage::PollExhausted, polling.elapsed());
#[cfg(feature = "scheduler-profiling")]
self.scheduler_profile
.record_poll(IDLE_SIGNAL_PROBES, false);
}
#[cfg(feature = "scheduler-profiling")]
self.scheduler_profile.record_wait(deadline.is_some());
{
#[cfg(feature = "handoff-profiling")]
let _publication = Span::new(HandoffStage::IdlePublication);
self.publish(CarrierStatus::Idle);
}
{
#[cfg(feature = "handoff-profiling")]
let _wait = Span::new(HandoffStage::WaitApi);
#[cfg(test)]
self.record_test_progress(crate::signal::TestCarrierPhase::Waiting);
let inbox = &self.inbox;
inbox.hub.wait_while(observed, deadline, || {
inbox.has_queued_starts_at_wait_boundary()
});
}
#[cfg(feature = "scheduler-profiling")]
self.scheduler_profile.record_wait_return(
self.inbox.pending() != 0
|| self.local.pending_wakes() != 0
|| self.inbox.hub.has_pending(),
);
}
fn observe_idle_work(&mut self) -> bool {
if self.inbox.pending() != 0 {
self.remote_pending = true;
true
} else {
self.local.pending_wakes() != 0 || self.inbox.hub.has_pending()
}
}
}
#[cfg(test)]
#[path = "kernel_receive_test.rs"]
mod kernel_receive_test;
#[cfg(test)]
#[path = "kernel_admission_test.rs"]
mod kernel_admission_test;