vthread 0.1.0

Bounded carrier-affine virtual threads for Rust
Documentation
//! Carrier-owned stacks, queues, and reclamation. This type is never Send.

#[path = "kernel_abort.rs"]
mod kernel_abort;
#[path = "kernel_cleanup.rs"]
mod kernel_cleanup;
#[path = "kernel_complete.rs"]
mod kernel_complete;
#[path = "kernel_drive.rs"]
mod kernel_drive;
#[path = "kernel_execution.rs"]
mod kernel_execution;
#[path = "kernel_receive.rs"]
mod kernel_receive;
#[path = "kernel_revoked.rs"]
mod kernel_revoked;
#[path = "kernel_task.rs"]
mod kernel_task;
#[path = "kernel_timer.rs"]
mod kernel_timer;
#[path = "kernel_wake.rs"]
mod kernel_wake;
#[path = "parked_tasks.rs"]
mod parked_tasks;

#[cfg(feature = "scheduler-profiling")]
use crate::scheduler_probe::SchedulerProfile;
use crate::{
    CarrierId, CarrierSnapshot, CarrierStatus, RuntimeStats, StackSnapshot,
    control::{CompletionBatch, CompletionUpdate, Shared},
    inbox::{Inbox, SpawnPacket},
    kernel_tasks::{KernelTasks, TaskMut, TaskRef},
    ready_queue::ReadyQueue,
    task_slab::TaskKey,
    timer::TimerQueue,
};
use crate::{context::Execution, local_carrier::LocalCarrier};
use parked_tasks::{ParkedTask, ParkedTasks};
use std::{cell::Cell, collections::VecDeque, rc::Rc, sync::Arc};

const PROGRESS_PUBLICATION_BATCH: u8 = 64;
const COMPLETION_BATCH: usize = 64;

pub(crate) struct Kernel {
    pub(super) shared: Arc<Shared>,
    pub(crate) inbox: Arc<Inbox>,
    pub(super) id: CarrierId,
    pub(super) tasks: KernelTasks,
    pub(super) ready: ReadyQueue,
    parked: ParkedTasks,
    deferred_wakes: Vec<crate::wait::WakeNotice>,
    pending_aborts: Vec<(Option<u64>, crate::TaskFailure)>,
    pub(super) in_flight: Option<TaskKey>,
    pub(super) pending: Option<SpawnPacket>,
    pub(super) incoming: VecDeque<SpawnPacket>,
    remote_pending: bool,
    pub(super) completions: CompletionBatch,
    completion_progress: Option<(u64, Arc<crate::control::ScopeProgress>)>,
    execution_cache: Vec<Rc<Execution>>,
    pub(crate) local: Rc<LocalCarrier>,
    pub(super) timers: TimerQueue,
    pub(super) stats: RuntimeStats,
    #[cfg(feature = "scheduler-profiling")]
    pub(super) scheduler_profile: SchedulerProfile,
    pub(super) has_borrowed: bool,
    observed_borrowed_scope_epoch: u64,
    admission_pressure: u32,
    unpublished_transitions: Cell<u8>,
    #[cfg(test)]
    pub(super) revocation_inspections: usize,
}

impl Kernel {
    pub(crate) fn new(shared: Arc<Shared>, id: CarrierId) -> Self {
        let config = shared.config;
        Self {
            inbox: Arc::clone(&shared.inboxes[id.0]),
            shared,
            id,
            tasks: KernelTasks::new(),
            ready: ReadyQueue::new(),
            parked: ParkedTasks::new(),
            deferred_wakes: Vec::new(),
            pending_aborts: Vec::new(),
            in_flight: None,
            pending: None,
            incoming: VecDeque::new(),
            remote_pending: false,
            completions: CompletionBatch::new(),
            completion_progress: None,
            execution_cache: Vec::new(),
            local: Rc::new(LocalCarrier::new(config)),
            timers: TimerQueue::new(),
            stats: RuntimeStats::default(),
            #[cfg(feature = "scheduler-profiling")]
            scheduler_profile: SchedulerProfile::default(),
            has_borrowed: false,
            observed_borrowed_scope_epoch: 0,
            admission_pressure: 0,
            unpublished_transitions: Cell::new(0),
            #[cfg(test)]
            revocation_inspections: 0,
        }
    }

    #[cfg(test)]
    pub(crate) fn record_test_loop(&self, observed: u64, handled: Option<u64>) {
        self.inbox
            .signal
            .test_progress
            .record_loop(observed, handled);
        self.record_test_progress(crate::signal::TestCarrierPhase::Drive);
    }

    #[cfg(test)]
    pub(crate) fn record_test_progress(&self, phase: crate::signal::TestCarrierPhase) {
        self.inbox.signal.test_progress.record_state(
            phase,
            crate::signal::TestCarrierState {
                remote_pending: self.remote_pending,
                admission_pressure: self.admission_pressure,
                ready: self.ready.len(),
                incoming: self.incoming.len(),
                pending_task: self.pending.as_ref().map(|packet| packet.test_id.get()),
                completions: self.completions.len(),
                in_flight: self
                    .in_flight
                    .map(|task| self.task(task).execution().id.get()),
            },
        );
    }

    pub(crate) fn execution(&self, task: TaskKey) -> Rc<Execution> {
        Rc::clone(self.task(task).execution())
    }

    pub(super) fn task(&self, key: TaskKey) -> TaskRef<'_> {
        self.tasks.get(key).expect("live task key")
    }

    pub(super) fn task_mut(&mut self, key: TaskKey) -> TaskMut<'_> {
        self.tasks.get_mut(key).expect("live task key")
    }

    pub(super) fn select_ready(&mut self) {
        self.in_flight = if self.pending_aborts.is_empty() {
            self.ready.pop_front()
        } else {
            self.select_unblocked()
        };
        crate::context::set_carrier_runnable(
            !self.ready.is_empty()
                || self.remote_pending
                || self.inbox.pending() != 0
                || self.local.pending_starts() != 0,
        );
    }

    pub(super) fn remove_in_flight(&mut self) {
        let key = self.in_flight.take().expect("in-flight task key");
        assert!(self.tasks.remove(key), "live in-flight task");
        if key.is_borrowed() {
            self.refresh_borrowed();
        }
    }

    pub(super) fn queue_completion(&mut self, completion: CompletionUpdate) {
        assert!(
            self.completions
                .scope()
                .is_none_or(|queued| queued == completion.scope),
            "completion batch crossed scope ownership"
        );
        if self
            .completion_progress
            .as_ref()
            .is_none_or(|(scope, _)| *scope != completion.scope)
        {
            assert!(self.completions.is_empty(), "live completion scope changed");
            self.completion_progress = Some((
                completion.scope,
                self.shared.scope_progress(completion.scope),
            ));
        }
        self.completions.push(completion);
        if self.completions.len() == COMPLETION_BATCH || !self.shared.may_defer_completion() {
            self.flush_completions();
        }
    }

    pub(super) fn flush_completions(&mut self) {
        if self.completions.is_empty() {
            return;
        }
        let (_, progress) = self
            .completion_progress
            .as_ref()
            .expect("completion scope progress");
        self.shared.publish_completions(&self.completions, progress);
        self.completions.clear();
    }

    pub(crate) fn publish(&self, status: CarrierStatus) {
        self.unpublished_transitions.set(0);
        self.shared.publish(self.snapshot(status));
    }

    pub(super) fn publish_transition(&self) {
        let unpublished = self.unpublished_transitions.get() + 1;
        if unpublished == PROGRESS_PUBLICATION_BATCH {
            self.publish(CarrierStatus::Running);
        } else {
            self.unpublished_transitions.set(unpublished);
        }
    }

    pub(crate) fn retire(&self, status: CarrierStatus) {
        #[cfg(test)]
        assert_ne!(
            self.shared
                .carrier_fault
                .load(std::sync::atomic::Ordering::Acquire),
            2,
            "injected carrier retirement failure"
        );
        let mut snapshot = self.snapshot(status);
        snapshot.stacks.cached = 0;
        self.shared.publish(snapshot);
    }

    pub(crate) fn reclaimed(&self) -> bool {
        self.pending.is_none()
            && self.incoming.is_empty()
            && self.in_flight.is_none()
            && self.ready.is_empty()
            && self.parked.is_empty()
            && self.tasks.is_empty()
            && self.deferred_wakes.is_empty()
            && self.pending_aborts.is_empty()
            && self.completions.is_empty()
            && self.shared.carrier_progress[self.id.0].mounted().is_none()
            && self.local.pending_starts() == 0
            && self.local.pending_wakes() == 0
            && self.inbox.pending() == 0
    }

    fn snapshot(&self, status: CarrierStatus) -> CarrierSnapshot {
        assert_eq!(
            self.tasks.len(),
            self.ready.len() + self.parked.len() + usize::from(self.in_flight.is_some())
        );
        let mut stats = self.stats;
        stats.stale_wakes += self.inbox.hub.stale();
        CarrierSnapshot {
            id: self.id,
            status,
            active: self.tasks.len(),
            runnable: self.ready.len(),
            parked: self.parked.len(),
            timers: self.timers.active_count(),
            pending_starts: self.inbox.pending(),
            pending_wakes: self.local.pending_wakes()
                + self.inbox.hub.pending()
                + self.deferred_wakes.len(),
            stats,
            stacks: StackSnapshot::from(self.local.stacks.borrow().snapshot()),
            #[cfg(feature = "scheduler-profiling")]
            scheduler_profile: self.scheduler_profile,
            #[cfg(feature = "handoff-profiling")]
            handoff_profile: *self.local.handoff_profile.borrow(),
        }
    }
}

#[cfg(test)]
#[path = "kernel_test.rs"]
mod kernel_test;

#[cfg(test)]
#[path = "kernel_policy_test.rs"]
mod kernel_policy_test;

#[cfg(test)]
#[path = "kernel_cross_join_test.rs"]
mod kernel_cross_join_test;

#[cfg(test)]
#[path = "kernel_join_deadline_test.rs"]
mod kernel_join_deadline_test;