vthread 0.1.0

Bounded carrier-affine virtual threads for Rust
Documentation
//! Fair bounded materialization of remote starts and unbounded local borrows.

use super::Kernel;
use crate::{CarrierStatus, TaskFailure, TaskStatus, kernel_tasks::OwnedTask};
#[cfg(feature = "handoff-profiling")]
use crate::{
    handoff_profile::HandoffStage,
    handoff_span::{Span, duration},
};
use std::sync::Arc;
use vthread_stack::Fiber;

const REMOTE_READY_TARGET: usize = 64;
const REMOTE_ADMISSION_DISPATCH_BOUND: u32 = 65_536;
// Bridge short multi-carrier admission gaps without turning an idle carrier into a poller.
// The carrier performs at most 640 pause instructions before entering the signal wait.
const IDLE_SIGNAL_PROBES: usize = 640;
const SPINS_PER_SIGNAL_PROBE: usize = 1;

impl Kernel {
    pub(crate) fn receive(&mut self) -> bool {
        #[cfg(test)]
        self.record_test_progress(crate::signal::TestCarrierPhase::Receive);
        // The carrier checks once per drive iteration while backlog remains,
        // independent of the previous dispatch outcome. Keep this off task paths.
        self.admission_pressure += u32::from(self.remote_pending);
        let received = self.receive_local_tasks();
        let received = self.receive_remote_tasks() || received;
        if received {
            self.publish(CarrierStatus::Running);
        }
        self.remote_pending = self.inbox.pending() != 0;
        if !self.remote_pending {
            self.admission_pressure = 0;
        }
        self.remote_pending
    }

    #[cfg(test)]
    pub(crate) fn remote_pending(&self) -> bool {
        self.remote_pending
    }

    pub(crate) fn remote_receive_required(&self) -> bool {
        // Published depth is authoritative while this carrier is driving, even
        // before notification. The cached bit keeps batch draining hot.
        self.remote_pending || self.inbox.pending() != 0
    }

    pub(crate) fn receive_local(&mut self) {
        if self.receive_local_tasks() {
            self.publish(CarrierStatus::Running);
        }
    }

    fn receive_local_tasks(&mut self) -> bool {
        let mut received = false;
        while let Some(task) = self.local.pop_start() {
            received = true;
            self.has_borrowed = true;
            #[cfg(feature = "runtime-evidence")]
            self.shared
                .record(crate::diagnostics::evidence::RuntimeEventKind::QueueDepth {
                    carrier: self.id,
                    queue: crate::diagnostics::evidence::QueueKind::LocalStart,
                    depth: self.local.pending_starts(),
                    capacity: self.shared.config.carrier_queue_capacity(),
                });
            self.shared.transition(
                task.execution.as_ref().expect("task execution").record(),
                |record| record.status = TaskStatus::Ready,
            );
            self.ready.push_back(self.tasks.insert_borrowed(task));
        }
        received
    }

    fn receive_remote_tasks(&mut self) -> bool {
        assert!(self.incoming.is_empty(), "unprocessed start batch");
        let capacity = self.shared.config.carrier_queue_capacity();
        let target = REMOTE_READY_TARGET.min(capacity);
        let limit = if self.ready.len() <= target / 2 {
            target - self.ready.len()
        // Keep normal refill inside the hot window. A continuously full window
        // still admits one start per quota, independent of dispatch outcome.
        } else if self.admission_pressure >= REMOTE_ADMISSION_DISPATCH_BOUND {
            1
        } else {
            #[cfg(feature = "scheduler-profiling")]
            self.scheduler_profile.record_receive(None);
            return false;
        };
        let drained = self.inbox.drain_into(&mut self.incoming, limit);
        #[cfg(feature = "scheduler-profiling")]
        self.scheduler_profile.record_receive(Some(drained));
        if drained != 0 {
            self.admission_pressure = 0;
        }
        let mut received = false;
        while let Some(packet) = self.incoming.pop_front() {
            self.pending = Some(packet);
            #[cfg(feature = "lifecycle-profiling")]
            let stack_fiber_started = std::time::Instant::now();
            #[cfg(feature = "runtime-evidence")]
            let acquired = self.local.stacks.borrow_mut().acquire_identified();
            #[cfg(not(feature = "runtime-evidence"))]
            let acquired = self.local.stacks.borrow_mut().acquire();
            #[cfg(feature = "runtime-evidence")]
            let (stack_identity, stack) = match acquired {
                Ok(stack) => stack,
                Err(_) => {
                    self.discard_pending(TaskFailure::StackAllocation);
                    #[cfg(feature = "lifecycle-profiling")]
                    self.shared
                        .lifecycle_probe
                        .record_stack_fiber(stack_fiber_started.elapsed());
                    continue;
                }
            };
            #[cfg(not(feature = "runtime-evidence"))]
            let stack = match acquired {
                Ok(stack) => stack,
                Err(_) => {
                    self.discard_pending(TaskFailure::StackAllocation);
                    #[cfg(feature = "lifecycle-profiling")]
                    self.shared
                        .lifecycle_probe
                        .record_stack_fiber(stack_fiber_started.elapsed());
                    continue;
                }
            };
            let packet = self.pending.as_mut().expect("pending packet");
            received = true;
            let entry = packet.entry.take().expect("unstarted packet entry");
            let body_record = Arc::clone(&packet.record);
            let fiber = Fiber::new(stack, move || entry.run(&body_record));
            #[cfg(feature = "runtime-evidence")]
            let task_fiber = crate::task_fiber::OwnedFiber::new(fiber, stack_identity);
            #[cfg(not(feature = "runtime-evidence"))]
            let task_fiber = crate::task_fiber::OwnedFiber::new(fiber);
            let (id, scope, options) = self.shared.transition(&packet.record, |record| {
                record.status = TaskStatus::Ready;
                (record.id, record.scope, record.options().clone())
            });
            let record = Arc::clone(&packet.record);
            let execution = self.acquire_execution(id, scope, record, options);
            let task = OwnedTask {
                fiber: Some(task_fiber),
                execution: Some(execution),
            };
            #[cfg(feature = "runtime-evidence")]
            self.shared.record(
                crate::diagnostics::evidence::RuntimeEventKind::StackCheckedOut {
                    task: id,
                    stack: crate::diagnostics::evidence::StackId::new(self.id, stack_identity),
                },
            );
            self.pending = None;
            self.ready.push_back(self.tasks.insert_owned(task));
            #[cfg(feature = "lifecycle-profiling")]
            self.shared
                .lifecycle_probe
                .record_stack_fiber(stack_fiber_started.elapsed());
        }
        received
    }

    pub(crate) fn wait_for_work(&mut self, observed: u64) {
        #[cfg(test)]
        self.record_test_progress(crate::signal::TestCarrierPhase::Idle);
        #[cfg(feature = "handoff-profiling")]
        let _episode = Span::new(HandoffStage::IdleEpisode);
        #[cfg(feature = "scheduler-profiling")]
        self.scheduler_profile.record_idle(self.stats.mounts);
        self.flush_completions();
        let deadline = self.timers.next_deadline();
        if deadline.is_some() {
            self.stats.timer_sleeps += 1;
        }
        if self.observe_idle_work() {
            #[cfg(feature = "scheduler-profiling")]
            self.scheduler_profile.record_early_work();
            return;
        }
        if deadline.is_none() && self.shared.config.carriers() > 1 {
            #[cfg(feature = "handoff-profiling")]
            let polling = std::time::Instant::now();
            for _probe in 0..IDLE_SIGNAL_PROBES {
                for _ in 0..SPINS_PER_SIGNAL_PROBE {
                    std::hint::spin_loop();
                }
                let work = self.observe_idle_work();
                if work || self.inbox.signal.version() != observed {
                    #[cfg(feature = "handoff-profiling")]
                    duration(
                        if work {
                            HandoffStage::PollWork
                        } else {
                            HandoffStage::PollControl
                        },
                        polling.elapsed(),
                    );
                    #[cfg(feature = "scheduler-profiling")]
                    self.scheduler_profile.record_poll(_probe + 1, true);
                    return;
                }
            }
            #[cfg(feature = "handoff-profiling")]
            duration(HandoffStage::PollExhausted, polling.elapsed());
            #[cfg(feature = "scheduler-profiling")]
            self.scheduler_profile
                .record_poll(IDLE_SIGNAL_PROBES, false);
        }
        #[cfg(feature = "scheduler-profiling")]
        self.scheduler_profile.record_wait(deadline.is_some());
        {
            #[cfg(feature = "handoff-profiling")]
            let _publication = Span::new(HandoffStage::IdlePublication);
            self.publish(CarrierStatus::Idle);
        }
        {
            #[cfg(feature = "handoff-profiling")]
            let _wait = Span::new(HandoffStage::WaitApi);
            #[cfg(test)]
            self.record_test_progress(crate::signal::TestCarrierPhase::Waiting);
            let inbox = &self.inbox;
            inbox.hub.wait_while(observed, deadline, || {
                inbox.has_queued_starts_at_wait_boundary()
            });
        }
        #[cfg(feature = "scheduler-profiling")]
        self.scheduler_profile.record_wait_return(
            self.inbox.pending() != 0
                || self.local.pending_wakes() != 0
                || self.inbox.hub.has_pending(),
        );
    }

    fn observe_idle_work(&mut self) -> bool {
        if self.inbox.pending() != 0 {
            // Queue publication precedes the coalesced signal. Remember visible
            // starts so the next drive receives them even if the notifier pauses.
            self.remote_pending = true;
            true
        } else {
            self.local.pending_wakes() != 0 || self.inbox.hub.has_pending()
        }
    }
}

#[cfg(test)]
#[path = "kernel_receive_test.rs"]
mod kernel_receive_test;

#[cfg(test)]
#[path = "kernel_admission_test.rs"]
mod kernel_admission_test;