ax-task 0.8.0

//! Remotely observable runqueue and owner-work publication state.

use super::*;

mod deadline;
mod delivery;
mod idle_pull;
mod ktimer;
mod lifecycle;
mod load_summary;
mod owner;
mod run_queue;
mod scheduler;

pub(crate) use deadline::{
    CpuDeadlineActivityGuard, CpuDeadlineBase, CpuDeadlinePublicationGuard, CpuDeadlineReadGuard,
    CpuDeadlineState, DeadlineBaseGuardSource, KtimerClaimClass, SchedulerDeadlinePublicationState,
    SchedulerNonTimerDeadlines,
};
pub(crate) use delivery::PreparedMigrationDelivery;
pub(crate) use idle_pull::IdlePullReservation;
pub use lifecycle::CpuLifecycleState;
pub(crate) use lifecycle::{CpuRemotePublication, OwnedCpuRemotePublication};
pub(crate) use load_summary::RunQueueLoadPublication;
pub use owner::CpuLocalOwnerBorrow;
pub(in crate::sched::system::cpu) use run_queue::RqCurrentUpdate;
pub(crate) use run_queue::{
    CpuRunQueueState, EqualRtWakeAction, OwnerRqEnqueue, RunQueueDomainPublication,
    RunQueueGuardSource, WakePreemptionContext, WakePreemptionDecision,
};
pub(crate) use scheduler::{RescheduleKind, SchedulerRequestClaim, SchedulerRequestScope};

/// Stable cross-CPU publication endpoint for one scheduler owner.
///
/// This object owns the IRQ-safe target runqueue, atomic delivery state, and
/// intrusive owner-control inboxes. Owner-only runtime accounting and switch
/// tail state remain in [`CpuLocal`].
#[derive(Debug)]
pub struct CpuRemote {
    owner: CpuId,
    pub(crate) migration_affinity: Arc<crate::sched::CpuSet>,
    run_queue: IrqTicketLock<CpuRunQueueState>,
    rt_bandwidth: IrqTicketLock<RtRunQueueBandwidth>,
    deadline: CpuDeadlineBase,
    /// Linux `dl_rq.extra_bw`: root-domain bandwidth published for this rq.
    deadline_extra_bw_scaled: AtomicU64,
    owner_state: owner::OwnerState,
    publication: lifecycle::CpuPublicationState,
    scheduler_request: scheduler::SchedulerRequestState,
    ktimer: ktimer::KtimerWorkerState,
    load: load_summary::RemoteLoadState,
    idle_pull: idle_pull::IdlePullState,
    delivery: delivery::RemoteDeliveryState,
}

impl CpuRemote {
    pub(crate) fn create(
        owner: CpuId,
        config: TaskSystemConfig,
    ) -> Result<Arc<Self>, crate::thread::TaskError> {
        let deadline_max_bw_scaled = u64::from(config.deadline_cap_percent())
            * crate::sched::algorithm::DEADLINE_UTILIZATION_SCALE
            / 100;
        let mut migration_affinity = crate::sched::CpuSet::empty(config.cpu_count());
        assert!(migration_affinity.insert(owner));
        crate::thread::allocation::try_arc(Self {
            owner,
            migration_affinity: Arc::new(migration_affinity),
            run_queue: IrqTicketLock::new(CpuRunQueueState::new(owner, config)?),
            rt_bandwidth: IrqTicketLock::new(RtRunQueueBandwidth::offline()),
            deadline: CpuDeadlineBase::new(config),
            deadline_extra_bw_scaled: AtomicU64::new(deadline_max_bw_scaled),
            owner_state: owner::OwnerState::new(),
            publication: lifecycle::CpuPublicationState::new(),
            scheduler_request: scheduler::SchedulerRequestState::new(),
            ktimer: ktimer::KtimerWorkerState::new(),
            load: load_summary::RemoteLoadState::new(),
            idle_pull: idle_pull::IdlePullState::new(),
            delivery: delivery::RemoteDeliveryState::new(),
        })
    }

    /// Acquires the target CPU runqueue with local IRQs disabled.
    ///
    /// Thread scheduler state must be acquired before this lock whenever one
    /// transaction needs both. Owner-only switch-tail state is never protected
    /// by this lock and must not escape its CPU-local scheduler baton.
    pub(crate) fn lock_run_queue(
        &self,
        source: RunQueueGuardSource,
    ) -> IrqTicketGuard<'_, CpuRunQueueState> {
        self.run_queue.lock(source.irq_guard_source())
    }

    /// Acquires this rq below an already-held task scheduler IRQ owner.
    pub(crate) fn lock_run_queue_nested<'a>(
        &'a self,
        owner: &'a IrqOwner<'_>,
    ) -> IrqTicketGuard<'a, CpuRunQueueState> {
        self.run_queue.lock_nested(owner)
    }

    /// Acquires the rq under an already-active IRQ-off CPU owner.
    ///
    /// # Safety
    ///
    /// The caller must retain either the scheduler baton or the offline boot
    /// CPU's Linux-style `PREEMPT_DISABLED` ownership, with local IRQs disabled
    /// for the complete guard lifetime. See
    /// [`IrqTicketLock::lock_irq_disabled`].
    pub(crate) unsafe fn lock_run_queue_irq_disabled(
        &self,
    ) -> IrqTicketGuard<'_, CpuRunQueueState> {
        // SAFETY: forwarded unchanged to the caller's scheduler-baton contract.
        unsafe { self.run_queue.lock_irq_disabled() }
    }

    /// Locks this CPU's hrtimer-style task-deadline base.
    ///
    /// The rq lock precedes this lock when both are required. Timer IRQ code
    /// takes only this lock; soft-timer callbacks release it before acquiring a
    /// task control lock or rq lock.
    pub(crate) fn read_deadline_base(
        &self,
        source: DeadlineBaseGuardSource,
    ) -> CpuDeadlineReadGuard<'_> {
        self.deadline.read(source)
    }

    /// Skips the IRQ-disabled deadline-base read when no timer, expiration, or
    /// softirq ownership has been published.
    pub(crate) fn read_active_deadline_base(
        &self,
        source: DeadlineBaseGuardSource,
    ) -> Option<CpuDeadlineReadGuard<'_>> {
        self.deadline.read_if_active(source)
    }

    /// Locks physical clockevent publication metadata.
    ///
    /// Publication changes neither the logical timer queue nor expiry
    /// ownership, so it must not rewrite the derived active bit.
    pub(crate) fn lock_deadline_publication(&self) -> CpuDeadlinePublicationGuard<'_> {
        self.deadline.lock_publication()
    }

    pub(crate) fn deadline_publication_snapshot_matches(
        &self,
        non_timer: SchedulerNonTimerDeadlines,
    ) -> bool {
        self.deadline.publication_snapshot_matches(non_timer)
    }

    /// Locks a transition that may change queue, buffered expiry, or softirq
    /// ownership and republishes the derived active bit before unlock.
    pub(crate) fn lock_deadline_activity(
        &self,
        source: DeadlineBaseGuardSource,
    ) -> CpuDeadlineActivityGuard<'_> {
        self.deadline.lock_activity(source)
    }

    /// Skips an expiry transition when the derived base publication is empty.
    pub(crate) fn lock_active_deadline_activity(
        &self,
        source: DeadlineBaseGuardSource,
    ) -> Option<CpuDeadlineActivityGuard<'_>> {
        self.deadline.lock_activity_if_active(source)
    }

    /// Locks Linux `rt_rq::rt_runtime_lock` after the owner rq lock when both
    /// are required. Fair-only rq transactions never enter this ledger.
    pub(crate) fn lock_rt_bandwidth(&self) -> IrqTicketGuard<'_, RtRunQueueBandwidth> {
        self.rt_bandwidth
            .lock(crate::runtime::IrqGuardSource::CpuRtBandwidthTicket)
    }

    pub(crate) fn publish_deadline_extra_bw(&self, extra_bw_scaled: u64) {
        self.deadline_extra_bw_scaled
            .store(extra_bw_scaled, Ordering::Release);
    }

    pub(crate) fn deadline_extra_bw_scaled(&self) -> u64 {
        self.deadline_extra_bw_scaled.load(Ordering::Acquire)
    }
}