use vthread_stack::ParkToken;
use super::{ResourceSelection, WaitCell, WakeCause, wait_state};
use crate::{Error, Result};
impl WaitCell {
pub(crate) fn reset_closed_gate(&self) -> bool {
let mut word = self.state.load();
loop {
if word.phase() != wait_state::Phase::Idle {
return false;
}
if !word.is_closed() {
return true;
}
match self.state.compare_exchange(word, word.with_closed(false)) {
Ok(()) => return true,
Err(observed) => word = observed,
}
}
}
pub(crate) fn finish(&self, token: ParkToken) -> Result<WakeCause> {
if token.wait() != self.state.id {
return Err(resumed_generation_fault());
}
self.finish_slow(token)
}
pub(crate) fn finish_plain_ready(&self, token: ParkToken) -> Result<WakeCause> {
self.finish_ready(token, None)
}
#[inline]
pub(crate) fn finish_permit_ready(&self, token: ParkToken) -> Result<WakeCause> {
if token.wait() != self.state.id {
return Err(resumed_generation_fault());
}
let selected = wait_state::WaitWord::initial()
.with_generation(token.generation())
.with_resource(Some(ResourceSelection::Permit))
.with_phase(wait_state::Phase::SelectedReady);
if self
.state
.compare_exchange(selected, selected.with_resource(None).retire())
.is_ok()
{
#[cfg(feature = "runtime-evidence")]
self.record_resumed(selected, token, WakeCause::Ready);
return Ok(WakeCause::Ready);
}
self.finish_permit_slow(token)
}
#[inline]
fn finish_ready(
&self,
token: ParkToken,
resource: Option<ResourceSelection>,
) -> Result<WakeCause> {
if token.wait() != self.state.id {
return Err(resumed_generation_fault());
}
let selected = wait_state::WaitWord::initial()
.with_generation(token.generation())
.with_resource(resource)
.with_phase(wait_state::Phase::SelectedReady);
if self
.state
.compare_exchange(selected, selected.retire())
.is_ok()
{
#[cfg(feature = "runtime-evidence")]
self.record_resumed(selected, token, WakeCause::Ready);
return Ok(WakeCause::Ready);
}
self.finish_slow(token)
}
fn finish_slow(&self, token: ParkToken) -> Result<WakeCause> {
#[cfg(feature = "handoff-profiling")]
let mut claim_span = None;
loop {
let word = self.state.load();
if word.generation() != token.generation() {
return Err(resumed_generation_fault());
}
if word.is_claimed() || word.phase() == wait_state::Phase::Binding {
#[cfg(feature = "handoff-profiling")]
claim_span.get_or_insert_with(|| {
crate::handoff_span::Span::new(
crate::handoff_profile::HandoffStage::ClaimFinish,
)
});
#[cfg(test)]
self.state.observe_publication(
super::wait_publication_probe_test::Stage::FinishWaiting,
token,
);
std::hint::spin_loop();
continue;
}
let Some(cause) = word.selected_cause() else {
return Err(Error::fault(
crate::error::FaultComponent::Scheduler,
"resumed parker has no selected wake",
));
};
if self.state.compare_exchange(word, word.retire()).is_ok() {
#[cfg(feature = "runtime-evidence")]
self.record_resumed(word, token, cause);
return Ok(cause);
}
}
}
#[cold]
#[inline(never)]
fn finish_permit_slow(&self, token: ParkToken) -> Result<WakeCause> {
#[cfg(feature = "handoff-profiling")]
let mut claim_span = None;
loop {
let word = self.state.load();
if word.generation() != token.generation() {
return Err(resumed_generation_fault());
}
if word.is_claimed() || word.phase() == wait_state::Phase::Binding {
#[cfg(feature = "handoff-profiling")]
claim_span.get_or_insert_with(|| {
crate::handoff_span::Span::new(
crate::handoff_profile::HandoffStage::ClaimFinish,
)
});
#[cfg(test)]
self.state.observe_publication(
super::wait_publication_probe_test::Stage::FinishWaiting,
token,
);
std::hint::spin_loop();
continue;
}
let Some(cause) = word.selected_cause() else {
return Err(Error::fault(
crate::error::FaultComponent::Scheduler,
"resumed parker has no selected wake",
));
};
let retired = if cause == WakeCause::Ready {
word.with_resource(None).retire()
} else {
word.retire()
};
if self.state.compare_exchange(word, retired).is_ok() {
#[cfg(feature = "runtime-evidence")]
self.record_resumed(word, token, cause);
return Ok(cause);
}
}
}
#[cfg(feature = "runtime-evidence")]
fn record_resumed(&self, word: wait_state::WaitWord, token: ParkToken, cause: WakeCause) {
let (task, evidence) = self
.state
.with_target(word, |task, _, hub| (task, hub.evidence()));
if let Some(evidence) = evidence {
evidence.record(crate::diagnostics::evidence::RuntimeEventKind::Resumed {
task,
wait: crate::diagnostics::evidence::WaitKey::from_token(token),
cause: cause.evidence(),
});
}
}
pub(crate) fn rollback(&self, token: ParkToken) {
let Some(hub) = self.state.retire(token) else {
return;
};
crate::context::unregister_local_wake(&hub, token);
hub.discard_notice(token);
}
}
fn resumed_generation_fault() -> Error {
Error::fault(
crate::error::FaultComponent::Scheduler,
"resumed parker generation changed",
)
}
#[cfg(test)]
#[path = "wait_finish_test.rs"]
mod wait_finish_test;