pub mod cancel;
mod harness;
pub mod registry;
mod runtime;
pub use cancel::{Cancel, Ctx};
pub(crate) use harness::FallibleClock;
pub use harness::{
AbortResult, Conversation, Harness, HarnessOptions, Inspection, Scheduling, TaskInspection, TaskInspectionState,
safe_now,
};
pub use registry::{Extension, Migrate, Registry, RegistryReader, RegistrySnapshot, Task};
pub use runtime::{NextState, TaskRuntime, aborted_with, completed};
use std::collections::{HashMap, HashSet};
use std::fmt;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Condvar, Mutex, MutexGuard, Weak};
use durable::{
AbortReason, CommitChange, CommitPublication, ConversationId, JoinPolicy, Session, SessionError, StorageWrite,
SubmissionQuery, SubmissionStatus, SubmissionType, TaskId, TaskOutcome, TaskOutcomeError, TaskQuery, TaskRecord,
TaskState, TaskStatus, Tx, TxScope,
};
use indexmap::{IndexMap, IndexSet};
use serde_json::Value;
use cancel::{Waiters, lock, wait_any};
use registry::phase_of;
const SCAN_PAGE_SIZE: usize = 256;
const LIVE_STATUSES: [TaskStatus; 4] = [
TaskStatus::Pending,
TaskStatus::Running,
TaskStatus::Waiting,
TaskStatus::Completing,
];
#[derive(Clone, Debug, PartialEq)]
pub enum HarnessError {
Session(SessionError),
Aborted,
Error(String),
Caused { message: String, cause: String },
}
impl HarnessError {
pub fn name(&self) -> &'static str {
match self {
HarnessError::Session(error) => error.name(),
HarnessError::Aborted => "AbortError",
HarnessError::Error(_) | HarnessError::Caused { .. } => "Error",
}
}
pub fn message(&self) -> String {
self.to_string()
}
pub fn cause(&self) -> Option<&str> {
match self {
HarnessError::Caused { cause, .. } => Some(cause),
_ => None,
}
}
pub fn msg(message: impl Into<String>) -> HarnessError {
HarnessError::Error(message.into())
}
fn into_session(self) -> SessionError {
match self {
HarnessError::Session(error) => error,
other => SessionError::Error(other.to_string()),
}
}
}
impl fmt::Display for HarnessError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
HarnessError::Session(error) => error.fmt(f),
HarnessError::Aborted => f.write_str("This operation was aborted"),
HarnessError::Error(message) | HarnessError::Caused { message, .. } => f.write_str(message),
}
}
}
impl std::error::Error for HarnessError {}
impl From<SessionError> for HarnessError {
fn from(error: SessionError) -> Self {
HarnessError::Session(error)
}
}
pub type Result<T> = std::result::Result<T, HarnessError>;
pub(crate) fn commit_on<T>(
session: &Session,
scope: TxScope,
change: impl FnOnce(&mut Tx<'_>) -> Result<T>,
) -> Result<T> {
let mut own = None;
let result = session.commit_with(scope, |tx| match change(tx) {
Ok(value) => Ok(value),
Err(error) => {
let carried = error.clone().into_session();
own = Some(error);
Err(carried)
}
});
result.map_err(|error| own.take().unwrap_or(HarnessError::Session(error)))
}
pub(crate) fn closed_error(session: &Session) -> HarnessError {
match session.failure() {
Some(failure) => HarnessError::Session(SessionError::Failed(Box::new(failure))),
None => HarnessError::msg("Harness is closed"),
}
}
pub(crate) fn contained<T>(call: impl FnOnce() -> Result<T>) -> Result<T> {
match catch_unwind(AssertUnwindSafe(call)) {
Ok(result) => result,
Err(panic) => {
let message = panic
.downcast_ref::<&str>()
.map(|text| (*text).to_owned())
.or_else(|| panic.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "[panic]".to_owned());
Err(HarnessError::Error(message))
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum SchedulerOutcome {
Faulted { message: String },
Orphaned { reason: String },
}
impl SchedulerOutcome {
fn outcome(&self) -> TaskOutcome {
match self {
SchedulerOutcome::Faulted { message } => TaskOutcome::Faulted {
error: TaskOutcomeError {
message: message.clone(),
detail: None,
},
},
SchedulerOutcome::Orphaned { reason } => TaskOutcome::Orphaned { reason: reason.clone() },
}
}
}
pub(crate) type SettleOutcome = Arc<dyn Fn(&mut Tx<'_>, &TaskRecord, &SchedulerOutcome) -> Result<()> + Send + Sync>;
pub(crate) type WithdrawInputs = Arc<dyn Fn(&mut Tx<'_>, ConversationId) -> Result<()> + Send + Sync>;
pub type Report = Arc<dyn Fn(&HarnessError) + Send + Sync>;
pub type Clock = Arc<dyn Fn() -> u64 + Send + Sync>;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Mode {
Run,
Abort,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct TaskNode {
conversation_id: ConversationId,
owner: Option<TaskId>,
background: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Up {
Task(TaskId),
Conversation(ConversationId),
}
enum Step {
Task(TaskId, TaskNode),
Conversation(ConversationId),
Unknown,
}
#[derive(Default)]
struct Overlay {
tasks: IndexMap<TaskId, TaskRecord>,
edges: HashMap<ConversationId, Option<TaskId>>,
}
#[derive(Clone, Copy)]
enum Scope {
Conversation(ConversationId),
Roots,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum BlockedReason {
MissingTask,
TaskTooOld,
MigrationFailed,
}
impl BlockedReason {
pub fn as_str(self) -> &'static str {
match self {
BlockedReason::MissingTask => "missing_task",
BlockedReason::TaskTooOld => "task_too_old",
BlockedReason::MigrationFailed => "migration_failed",
}
}
}
enum Fit {
Ready {
task: Task,
migrates: bool,
},
Blocked {
reason: BlockedReason,
error: Option<HarnessError>,
},
}
enum Resolution {
Ready { task: Task, record: Box<TaskRecord> },
Blocked(BlockedReason),
}
fn node_of(record: &TaskRecord) -> TaskNode {
TaskNode {
conversation_id: record.conversation_id,
owner: record.owner,
background: record.background,
}
}
fn parent_of(node: &TaskNode) -> Up {
match node.owner {
Some(owner) => Up::Task(owner),
None => Up::Conversation(node.conversation_id),
}
}
fn record_parent(record: &TaskRecord) -> Up {
parent_of(&node_of(record))
}
fn outcome_of(record: &TaskRecord) -> Option<&TaskOutcome> {
match &record.state {
TaskState::Completing { outcome } | TaskState::Terminal { outcome } => Some(outcome),
_ => None,
}
}
fn failed_outcome(record: &TaskRecord) -> bool {
outcome_of(record).is_some_and(|outcome| !matches!(outcome, TaskOutcome::Completed { .. }))
}
fn cancellation_intent(record: &TaskRecord) -> bool {
record.state.status() != TaskStatus::Terminal && (record.abort_requested || failed_outcome(record))
}
fn with_abort_mark(record: &TaskRecord, reason: Option<AbortReason>) -> TaskRecord {
TaskRecord {
abort_requested: true,
abort_reason: reason,
..record.clone()
}
}
pub(crate) fn with_state(record: &TaskRecord, state: TaskState) -> TaskRecord {
let memos = match state {
TaskState::Terminal { .. } | TaskState::Completing { .. } => None,
_ => record.memos.clone(),
};
TaskRecord {
state,
memos,
..record.clone()
}
}
fn checkpoint_of(record: &TaskRecord) -> Option<&Value> {
match &record.state {
TaskState::Pending { checkpoint }
| TaskState::Running { checkpoint }
| TaskState::Waiting { checkpoint, .. } => Some(checkpoint),
_ => None,
}
}
fn status_name(status: TaskStatus) -> &'static str {
match status {
TaskStatus::Pending => "pending",
TaskStatus::Running => "running",
TaskStatus::Waiting => "waiting",
TaskStatus::Completing => "completing",
TaskStatus::Terminal => "terminal",
}
}
fn can_reserve(task: &Task, record: &TaskRecord) -> bool {
task.version() == record.version || (task.version() > record.version && task.migration().is_some())
}
fn missing_migration(record: &TaskRecord, task: &Task) -> HarnessError {
HarnessError::Error(format!(
"Task {} version {} has no migration from {}",
record.kind,
task.version(),
record.version
))
}
pub(crate) fn json_equal(left: &Value, right: &Value) -> bool {
match (left, right) {
(Value::Number(a), Value::Number(b)) => a == b || a.as_f64() == b.as_f64(),
(Value::Array(a), Value::Array(b)) => a.len() == b.len() && a.iter().zip(b).all(|(x, y)| json_equal(x, y)),
(Value::Object(a), Value::Object(b)) => {
a.len() == b.len()
&& a.iter()
.all(|(key, value)| b.get(key).is_some_and(|other| json_equal(value, other)))
}
_ => left == right,
}
}
pub(crate) struct Invocation {
pub(crate) task_id: TaskId,
pub(crate) conversation_id: ConversationId,
pub(crate) mode: Mode,
pub(crate) controller: Cancel,
ended: AtomicBool,
pub(crate) done: Cancel,
}
impl Invocation {
fn ended_error(&self) -> HarnessError {
HarnessError::Error(format!("Task {} invocation has ended", self.task_id.0))
}
pub(crate) fn check(&self) -> Result<()> {
if self.is_ended() {
return Err(self.ended_error());
}
Ok(())
}
pub(crate) fn is_ended(&self) -> bool {
self.ended.load(Ordering::SeqCst)
}
}
pub(crate) struct PhaseState {
pub(crate) task: Task,
pub(crate) snapshot: Arc<RegistrySnapshot>,
reported: Option<Option<Task>>,
}
struct Reservation {
invocation: Arc<Invocation>,
task: Task,
snapshot: Arc<RegistrySnapshot>,
}
struct PhaseResult {
checkpoint: Value,
failure: Option<HarnessError>,
}
enum Decision {
Continue,
End,
Fault(HarnessError),
}
#[derive(Default)]
struct State {
live: IndexMap<TaskId, TaskRecord>,
invocations: HashMap<TaskId, Arc<Invocation>>,
failed_migrations: HashMap<TaskId, (Task, HarnessError)>,
edges: HashMap<ConversationId, Option<TaskId>>,
conversation_owners: HashSet<TaskId>,
settled: HashMap<TaskId, TaskNode>,
fail_fast_checks: IndexSet<TaskId>,
abandoned: IndexSet<TaskId>,
reconcile_scheduled: bool,
cascade_pending: bool,
enabled: bool,
closing: bool,
dirty: bool,
}
impl State {
fn schedule_reconcile(&mut self) {
if !self.reconcile_scheduled && !self.closing {
self.reconcile_scheduled = true;
}
}
fn set_edge(&mut self, conversation_id: ConversationId, owner: Option<TaskId>) {
self.edges.insert(conversation_id, owner);
if let Some(owner) = owner {
self.conversation_owners.insert(owner);
}
}
fn edge(&self, id: ConversationId, overlay: Option<&Overlay>) -> Option<Option<TaskId>> {
if let Some(edge) = overlay.and_then(|overlay| overlay.edges.get(&id)) {
return Some(*edge);
}
self.edges.get(&id).copied()
}
fn node(&self, id: TaskId, overlay: Option<&Overlay>) -> Option<TaskNode> {
overlay
.and_then(|overlay| overlay.tasks.get(&id))
.or_else(|| self.live.get(&id))
.map(node_of)
.or_else(|| self.settled.get(&id).copied())
}
fn above(&self, start: Up, overlay: Option<&Overlay>) -> Vec<Step> {
let mut steps = Vec::new();
let mut at = Some(start);
while let Some(up) = at {
match up {
Up::Task(id) => {
let Some(node) = self.node(id, overlay) else {
steps.push(Step::Unknown);
return steps;
};
steps.push(Step::Task(id, node));
at = Some(parent_of(&node));
}
Up::Conversation(id) => {
steps.push(Step::Conversation(id));
match self.edge(id, overlay) {
None => {
steps.push(Step::Unknown);
return steps;
}
Some(edge) => at = edge.map(Up::Task),
}
}
}
}
steps
}
fn chain_known(&self, start: Up, overlay: Option<&Overlay>) -> bool {
!self
.above(start, overlay)
.iter()
.any(|step| matches!(step, Step::Unknown))
}
fn live_records<'a>(&'a self, overlay: Option<&'a Overlay>) -> Vec<&'a TaskRecord> {
let mut records = Vec::new();
for record in self.live.values() {
let candidate = overlay
.and_then(|overlay| overlay.tasks.get(&record.id))
.unwrap_or(record);
if candidate.state.status() != TaskStatus::Terminal {
records.push(candidate);
}
}
if let Some(overlay) = overlay {
for record in overlay.tasks.values() {
if !self.live.contains_key(&record.id) && record.state.status() != TaskStatus::Terminal {
records.push(record);
}
}
}
records
}
fn owned_live(&self, overlay: Option<&Overlay>) -> HashMap<TaskId, Vec<TaskId>> {
let mut owned: HashMap<TaskId, Vec<TaskId>> = HashMap::new();
for record in self.live_records(overlay) {
if record.background {
continue;
}
for step in self.above(record_parent(record), overlay) {
match step {
Step::Unknown => break,
Step::Conversation(_) => continue,
Step::Task(id, node) => {
owned.entry(id).or_default().push(record.id);
if node.background {
break;
}
}
}
}
}
owned
}
fn in_scope(&self, start: Up, scope: Scope, cross_background: bool) -> Option<bool> {
for step in self.above(start, None) {
match step {
Step::Unknown => return None,
Step::Conversation(id) => {
if matches!(scope, Scope::Conversation(target) if target == id) {
return Some(true);
}
}
Step::Task(_, node) => {
if node.background && !cross_background {
return Some(false);
}
}
}
}
Some(matches!(scope, Scope::Roots))
}
fn cancelling_owner(&self, start: Up) -> Option<&TaskRecord> {
for step in self.above(start, None) {
match step {
Step::Unknown => return None,
Step::Conversation(_) => continue,
Step::Task(id, node) => {
if let Some(live) = self.live.get(&id)
&& cancellation_intent(live)
{
return Some(live);
}
if node.background {
return None;
}
}
}
}
None
}
fn below_cancelled(&self, start: Up) -> bool {
self.cancelling_owner(start).is_some()
}
fn idle(&self, conversation_id: Option<ConversationId>) -> bool {
let scope = match conversation_id {
Some(id) => Scope::Conversation(id),
None => Scope::Roots,
};
!self
.live
.values()
.any(|record| !record.background && self.in_scope(record_parent(record), scope, false) != Some(false))
}
fn waiting_on(&self, record: &TaskRecord, owned: &HashMap<TaskId, Vec<TaskId>>) -> Vec<TaskId> {
if record.abort_requested {
return owned.get(&record.id).cloned().unwrap_or_default();
}
match &record.state {
TaskState::Waiting { on, .. } => on.iter().copied().filter(|id| self.live.contains_key(id)).collect(),
_ => Vec::new(),
}
}
fn fit(&self, record: &TaskRecord, task: Option<&Task>) -> Fit {
let Some(task) = task else {
return Fit::Blocked {
reason: BlockedReason::MissingTask,
error: None,
};
};
if task.version() == record.version {
return Fit::Ready {
task: task.clone(),
migrates: false,
};
}
if task.version() < record.version {
return Fit::Blocked {
reason: BlockedReason::TaskTooOld,
error: None,
};
}
if let Some((failed, error)) = self.failed_migrations.get(&record.id)
&& failed.same(task)
{
return Fit::Blocked {
reason: BlockedReason::MigrationFailed,
error: Some(error.clone()),
};
}
Fit::Ready {
task: task.clone(),
migrates: true,
}
}
}
fn overlay_of(tx: &Tx<'_>) -> Overlay {
Overlay {
tasks: tx
.staged_tasks()
.into_iter()
.map(|record| (record.id, record))
.collect(),
edges: tx
.staged_conversations()
.into_iter()
.map(|record| (record.id, record.owner.map(|owner| owner.task_id)))
.collect(),
}
}
pub(crate) struct SchedulerOptions {
pub session: Session,
pub registry: Arc<dyn RegistryReader>,
pub now: Clock,
pub report: Report,
pub settle_outcome: SettleOutcome,
pub withdraw_inputs: WithdrawInputs,
}
pub(crate) struct Scheduler {
pub(crate) session: Session,
registry: Arc<dyn RegistryReader>,
pub(crate) now: Clock,
pub(crate) report: Report,
settle_outcome: SettleOutcome,
withdraw_inputs: WithdrawInputs,
state: Mutex<State>,
wake: Condvar,
task_waiters: Waiters<TaskId, TaskRecord>,
idle_waiters: Waiters<Option<ConversationId>, ()>,
failed: Cancel,
unsubscribe_registry: Mutex<Option<Box<dyn FnOnce() + Send>>>,
failure_reported: AtomicBool,
this: Weak<Scheduler>,
}
impl Scheduler {
pub(crate) fn new(options: SchedulerOptions) -> Arc<Scheduler> {
Arc::new_cyclic(|this| Scheduler {
session: options.session,
registry: options.registry,
now: options.now,
report: options.report,
settle_outcome: options.settle_outcome,
withdraw_inputs: options.withdraw_inputs,
state: Mutex::new(State::default()),
wake: Condvar::new(),
task_waiters: Waiters::new(),
idle_waiters: Waiters::new(),
failed: Cancel::new(),
unsubscribe_registry: Mutex::new(None),
failure_reported: AtomicBool::new(false),
this: this.clone(),
})
}
fn st(&self) -> MutexGuard<'_, State> {
lock(&self.state)
}
fn arc(&self) -> Arc<Scheduler> {
self.this.upgrade().expect("scheduler is alive while used")
}
pub(crate) fn is_closing(&self) -> bool {
self.st().closing
}
fn commit<T>(&self, change: impl FnOnce(&mut Tx<'_>) -> Result<T>) -> Result<T> {
commit_on(&self.session, TxScope::default(), change)
}
pub(crate) fn open(&self) -> Result<()> {
let weak = self.this.clone();
self.session.subscribe_commits(move |publication| {
if let Some(scheduler) = weak.upgrade() {
scheduler.observe(publication);
}
})?;
let weak = self.this.clone();
self.session.subscribe_close(move || {
if let Some(scheduler) = weak.upgrade() {
scheduler.seal();
}
})?;
let weak = self.this.clone();
let unsubscribe = self.registry.subscribe(Box::new(move || {
if let Some(scheduler) = weak.upgrade() {
scheduler.kick();
}
}));
*lock(&self.unsubscribe_registry) = Some(unsubscribe);
if self.is_closing() {
if let Some(unsubscribe) = lock(&self.unsubscribe_registry).take() {
unsubscribe();
}
}
self.commit(|tx| {
let mut scans = Vec::new();
for status in LIVE_STATUSES {
let query = TaskQuery {
status: Some(status),
..TaskQuery::default()
};
let mut cursor = None;
loop {
let page = tx.scan_tasks(&query, SCAN_PAGE_SIZE, cursor.as_ref())?;
scans.extend(page.items);
match page.next {
Some(next) => cursor = Some(next),
None => break,
}
}
}
let mut writes = Vec::new();
{
let mut st = self.st();
for record in scans {
if record.abandon_on_restart {
st.abandoned.insert(record.id);
}
if let TaskState::Running { checkpoint } = &record.state {
writes.push(with_state(
&record,
TaskState::Pending {
checkpoint: checkpoint.clone(),
},
));
}
if let TaskState::Waiting {
policy: JoinPolicy::FailFast,
..
} = &record.state
{
st.fail_fast_checks.insert(record.id);
}
st.live.insert(record.id, record);
}
}
for record in writes {
tx.set_task(record)?;
}
Ok(())
})?;
{
let mut st = self.st();
st.cascade_pending = true;
st.schedule_reconcile();
}
let scheduler = self.arc();
std::thread::Builder::new()
.name("durable-scheduler".into())
.spawn(move || scheduler.work())
.map_err(|error| HarnessError::Error(error.to_string()))?;
self.wake.notify_all();
Ok(())
}
pub(crate) fn resume(&self) {
self.st().enabled = true;
self.kick();
}
pub(crate) fn join(&self) {
let invocations: Vec<Arc<Invocation>> = self.st().invocations.values().cloned().collect();
for invocation in &invocations {
invocation.controller.abort();
}
for invocation in &invocations {
invocation.done.wait();
}
}
pub(crate) fn abort(&self, id: TaskId, ctx: &Ctx, keep_restart: bool) -> Result<AbortResult> {
let (result, run) = self.commit(|tx| {
ctx.check()?;
let Some(current) = tx.task(id)? else {
return Err(HarnessError::Error(format!("Task {} does not exist", id.0)));
};
if current.state.status() == TaskStatus::Terminal {
return Ok((AbortResult::Terminal, None));
}
let invocation = self.st().invocations.get(&id).cloned();
if keep_restart && current.abort_reason == Some(AbortReason::Restart) {
return Ok((AbortResult::Marked, None));
}
if invocation.is_none() && current.state.status() != TaskStatus::Completing {
self.load_scopes(tx, false)?;
let owned = self.st().owned_live(None).contains_key(&id);
if !owned {
let snapshot = self.registry.snapshot();
if let Resolution::Blocked(reason) = self.resolve(¤t, &snapshot) {
let outcome = SchedulerOutcome::Orphaned {
reason: reason.as_str().into(),
};
self.terminate(tx, ¤t, &outcome)?;
return Ok((AbortResult::Marked, None));
}
}
}
if !current.abort_requested || current.abort_reason.is_some() {
tx.set_task(with_abort_mark(¤t, None))?;
}
let run = invocation.filter(|invocation| invocation.mode == Mode::Run);
Ok((AbortResult::Marked, run))
})?;
if let Some(run) = run {
let caller = ctx.signal().cloned().unwrap_or_default();
let (index, reason) = wait_any(&[&run.done, &self.failed, &caller]);
if index != 0 {
return Err(reason);
}
}
Ok(result)
}
pub(crate) fn wait_for_task(&self, id: TaskId, ctx: &Ctx) -> Result<TaskRecord> {
let slot = self.commit(|tx| {
if self.is_closing() {
return Err(closed_error(&self.session));
}
if self.st().live.contains_key(&id) {
return self.task_waiters.add(id, ctx);
}
match tx.task(id)? {
Some(record) => Ok(cancel::Slot::settled(Ok(record))),
None => Err(HarnessError::Error(format!("Task {} does not exist", id.0))),
}
})?;
slot.wait(ctx)
}
pub(crate) fn wait_for_idle(&self, conversation_id: Option<ConversationId>, ctx: &Ctx) -> Result<()> {
let slot = {
let mut st = self.st();
if st.closing {
drop(st);
return Err(closed_error(&self.session));
}
if st.idle(conversation_id) {
return Ok(());
}
st.schedule_reconcile();
self.wake.notify_all();
self.idle_waiters.add(conversation_id, ctx)?
};
slot.wait(ctx)
}
pub(crate) fn abort_conversation(
&self,
conversation_id: ConversationId,
background: bool,
ctx: &Ctx,
) -> Result<()> {
let reached = self.commit(|tx| {
ctx.check()?;
let queued = self.load_scopes(tx, true)?;
let scope = Scope::Conversation(conversation_id);
let (reached, marks, withdraw) = {
let st = self.st();
let mut reached = Vec::new();
let mut marks = Vec::new();
for record in st.live.values() {
if record.background && !background {
continue;
}
if st.in_scope(record_parent(record), scope, background) != Some(true) {
continue;
}
reached.push(record.id);
if !record.abort_requested || record.abort_reason.is_some() {
marks.push(with_abort_mark(record, None));
}
}
let withdraw: Vec<ConversationId> = queued
.into_iter()
.filter(|id| st.in_scope(Up::Conversation(*id), scope, background) == Some(true))
.collect();
(reached, marks, withdraw)
};
for mark in marks {
tx.set_task(mark)?;
}
for id in withdraw {
(self.withdraw_inputs)(tx, id)?;
}
Ok(reached)
})?;
if background {
for id in reached {
self.wait_for_task(id, ctx)?;
}
}
self.wait_for_idle(Some(conversation_id), ctx)
}
fn observe(&self, publication: &CommitPublication) {
let mut signal = Vec::new();
let mut resolved = Vec::new();
let mut changed = false;
{
let mut st = self.st();
let mut updated = Vec::new();
let mut failed = Vec::new();
for change in &publication.changes {
let CommitChange::Table(StorageWrite::Task { value: record }) = change else {
continue;
};
changed = true;
let previous = st.live.get(&record.id).cloned();
if failed_outcome(record) && previous.as_ref().is_none_or(|previous| !failed_outcome(previous)) {
failed.push(record.id);
}
if record.state.status() == TaskStatus::Terminal {
st.live.shift_remove(&record.id);
st.failed_migrations.remove(&record.id);
st.fail_fast_checks.shift_remove(&record.id);
if st.conversation_owners.contains(&record.id) {
st.settled.insert(record.id, node_of(record));
}
resolved.push(record.clone());
st.schedule_reconcile();
continue;
}
if record.abort_requested && previous.as_ref().is_none_or(|previous| !previous.abort_requested) {
st.cascade_pending = true;
if let Some(invocation) = st.invocations.get(&record.id)
&& invocation.mode == Mode::Run
{
signal.push(invocation.controller.clone());
}
}
if previous
.as_ref()
.is_some_and(|previous| previous.abort_reason.is_some())
&& record.abort_reason.is_none()
{
st.cascade_pending = true;
}
let status = record.state.status();
let previous_status = previous.as_ref().map(|previous| previous.state.status());
if status == TaskStatus::Completing && previous_status != Some(TaskStatus::Completing) {
if cancellation_intent(record) {
st.cascade_pending = true;
}
st.schedule_reconcile();
}
if let TaskState::Waiting {
policy: JoinPolicy::FailFast,
..
} = record.state
&& previous_status != Some(TaskStatus::Waiting)
{
st.fail_fast_checks.insert(record.id);
st.schedule_reconcile();
}
st.live.insert(record.id, record.clone());
updated.push(record.clone());
}
for id in failed {
let waiters: Vec<TaskId> = st
.live
.values()
.filter(|record| {
matches!(&record.state, TaskState::Waiting { policy: JoinPolicy::FailFast, on, .. } if on.contains(&id))
})
.map(|record| record.id)
.collect();
for waiter in waiters {
st.fail_fast_checks.insert(waiter);
st.schedule_reconcile();
}
}
for change in &publication.changes {
if let CommitChange::Table(StorageWrite::Conversation { value }) = change
&& !st.edges.contains_key(&value.id)
{
st.set_edge(value.id, value.owner.map(|owner| owner.task_id));
}
}
for change in &publication.changes {
let CommitChange::Table(StorageWrite::Submission { value }) = change else {
continue;
};
if value.status != SubmissionStatus::Queued || value.kind != SubmissionType::Input {
continue;
}
let up = Up::Conversation(value.conversation_id);
if !st.chain_known(up, None) || st.below_cancelled(up) {
st.cascade_pending = true;
}
}
for record in &updated {
let parent = record_parent(record);
if !st.chain_known(parent, None) {
st.schedule_reconcile();
} else if !record.background && !record.abort_requested && st.below_cancelled(parent) {
st.cascade_pending = true;
}
}
if st.cascade_pending {
st.schedule_reconcile();
}
}
self.wake.notify_all();
for controller in signal {
controller.abort();
}
for record in resolved {
self.task_waiters.resolve(&record.id, record.clone());
}
if !changed {
return;
}
self.settle_idle();
self.kick();
}
fn settle_idle(&self) {
let idle: Vec<Option<ConversationId>> = {
let st = self.st();
self.idle_waiters
.keys()
.into_iter()
.filter(|conversation_id| st.idle(*conversation_id))
.collect()
};
for conversation_id in idle {
self.idle_waiters.resolve(&conversation_id, ());
}
}
fn kick(&self) {
self.st().dirty = true;
self.wake.notify_all();
}
fn work(self: Arc<Self>) {
enum Job {
Reconcile,
Reserve,
}
loop {
let job = {
let mut st = self.st();
loop {
if st.closing {
return;
}
if st.reconcile_scheduled {
break Job::Reconcile;
}
if st.dirty && st.enabled {
st.dirty = false;
break Job::Reserve;
}
st = self.wake.wait(st).unwrap_or_else(|poisoned| poisoned.into_inner());
}
};
match job {
Job::Reconcile => self.reconcile(),
Job::Reserve => match self.reserve() {
Ok(reservations) => {
for reservation in reservations {
self.start(reservation);
}
}
Err(error) => self.fail_session(error),
},
}
}
}
fn reconcile(&self) {
let (cascade, checks) = {
let mut st = self.st();
st.reconcile_scheduled = false;
let cascade = std::mem::take(&mut st.cascade_pending);
let checks: Vec<TaskId> = std::mem::take(&mut st.fail_fast_checks).into_iter().collect();
(cascade, checks)
};
let result = self.commit(|tx| {
if self.is_closing() {
return Ok(());
}
let queued = self.load_scopes(tx, cascade)?;
let mut marks: IndexMap<TaskId, (TaskRecord, Option<AbortReason>)> = IndexMap::new();
fn mark(
marks: &mut IndexMap<TaskId, (TaskRecord, Option<AbortReason>)>,
record: &TaskRecord,
reason: Option<AbortReason>,
) {
if let Some((_, staged)) = marks.get(&record.id)
&& staged.is_none()
{
return;
}
if record.abort_requested && (record.abort_reason.is_none() || reason.is_some()) {
return;
}
marks.insert(record.id, (record.clone(), reason));
}
let fail_fast: Vec<(TaskId, Vec<TaskId>)> = {
let st = self.st();
for record in st.live.values() {
if record.background {
continue;
}
let Some(owner) = st.cancelling_owner(record_parent(record)) else {
continue;
};
let restart = owner.abort_reason == Some(AbortReason::Restart) && !failed_outcome(owner);
mark(&mut marks, record, restart.then_some(AbortReason::Restart));
}
checks
.iter()
.filter_map(|id| match st.live.get(id).map(|record| &record.state) {
Some(TaskState::Waiting { on, .. }) => Some((*id, on.clone())),
_ => None,
})
.collect()
};
for (_, on) in fail_fast {
if !self.any_failed(tx, &on)? {
continue;
}
let st = self.st();
for member in &on {
if let Some(record) = st.live.get(member)
&& !failed_outcome(record)
{
mark(&mut marks, record, None);
}
}
}
for (record, reason) in marks.values() {
tx.set_task(with_abort_mark(record, *reason))?;
}
let withdraw: Vec<ConversationId> = {
let st = self.st();
queued
.into_iter()
.filter(|id| st.below_cancelled(Up::Conversation(*id)))
.collect()
};
for id in withdraw {
(self.withdraw_inputs)(tx, id)?;
}
self.finalize(tx)
});
if let Err(error) = result {
self.fail_session(error);
}
self.settle_idle();
}
fn any_failed(&self, tx: &mut Tx<'_>, ids: &[TaskId]) -> Result<bool> {
for id in ids {
let live = self.st().live.get(id).cloned();
let record = match live {
Some(record) => Some(record),
None => tx.committed(|storage| storage.task(*id))?,
};
if record.is_some_and(|record| failed_outcome(&record)) {
return Ok(true);
}
}
Ok(false)
}
fn finalize(&self, tx: &mut Tx<'_>) -> Result<()> {
loop {
let overlay = overlay_of(tx);
let done: Vec<TaskRecord> = {
let st = self.st();
let owned = st.owned_live(Some(&overlay));
st.live_records(Some(&overlay))
.into_iter()
.filter(|record| record.state.status() == TaskStatus::Completing && !owned.contains_key(&record.id))
.cloned()
.collect()
};
if done.is_empty() {
return Ok(());
}
for record in done {
let TaskState::Completing { outcome } = &record.state else {
unreachable!("filtered")
};
let outcome = outcome.clone();
tx.set_task(with_state(
&record,
TaskState::Terminal {
outcome: outcome.clone(),
},
))?;
let scheduler_outcome = match outcome {
TaskOutcome::Faulted { error } => Some(SchedulerOutcome::Faulted { message: error.message }),
TaskOutcome::Orphaned { reason } => Some(SchedulerOutcome::Orphaned { reason }),
_ => None,
};
if let Some(outcome) = scheduler_outcome {
(self.settle_outcome)(tx, &record, &outcome)?;
}
}
}
}
fn load_scopes(&self, tx: &mut Tx<'_>, queued: bool) -> Result<Vec<ConversationId>> {
let parents: Vec<Up> = {
let st = self.st();
st.live
.values()
.map(record_parent)
.filter(|parent| !st.chain_known(*parent, None))
.collect()
};
for parent in parents {
self.load_chain(tx, parent, None)?;
}
if !queued {
return Ok(Vec::new());
}
let query = SubmissionQuery {
status: Some(SubmissionStatus::Queued),
..SubmissionQuery::default()
};
let mut conversations: IndexSet<ConversationId> = IndexSet::new();
let mut cursor = None;
loop {
let page = tx.committed(|storage| storage.scan_submissions(&query, SCAN_PAGE_SIZE, cursor.as_ref()))?;
conversations.extend(page.items.iter().map(|submission| submission.conversation_id));
match page.next {
Some(next) => cursor = Some(next),
None => break,
}
}
for id in &conversations {
self.load_chain(tx, Up::Conversation(*id), None)?;
}
Ok(conversations.into_iter().collect())
}
fn load_chain(&self, tx: &mut Tx<'_>, start: Up, overlay: Option<&Overlay>) -> Result<()> {
let mut at = Some(start);
while let Some(up) = at {
match up {
Up::Task(id) => {
let known = self.st().node(id, overlay);
let node = match known {
Some(node) => node,
None => {
let Some(record) = tx.committed(|storage| storage.task(id))? else {
return Ok(());
};
let node = node_of(&record);
if record.state.status() == TaskStatus::Terminal {
self.st().settled.insert(record.id, node);
}
node
}
};
at = Some(parent_of(&node));
}
Up::Conversation(id) => {
let known = self.st().edge(id, overlay);
let edge = match known {
Some(edge) => edge,
None => {
let record = tx.committed(|storage| storage.conversation(id))?;
let edge = record.and_then(|record| record.owner).map(|owner| owner.task_id);
self.st().set_edge(id, edge);
edge
}
};
at = edge.map(Up::Task);
}
}
}
Ok(())
}
fn fail_session(&self, error: HarnessError) {
if !self.is_closing() && !matches!(error, HarnessError::Session(SessionError::Closed)) {
self.session.fail(error.into_session());
}
}
fn seal(&self) {
{
let mut st = self.st();
st.closing = true;
}
self.wake.notify_all();
if let Some(unsubscribe) = lock(&self.unsubscribe_registry).take() {
unsubscribe();
}
let error = closed_error(&self.session);
self.task_waiters.reject_all(error.clone());
self.idle_waiters.reject_all(error.clone());
match self.session.failure() {
Some(_) => {
self.failed.cancel(error);
self.report_failure();
let invocations: Vec<Arc<Invocation>> = self.st().invocations.values().cloned().collect();
for invocation in invocations {
invocation.controller.abort();
}
}
None => self.join(),
}
}
pub(crate) fn report_failure(&self) {
if let Some(failure) = self.session.failure()
&& !self.failure_reported.swap(true, Ordering::SeqCst)
{
self.failed.cancel(closed_error(&self.session));
(self.report)(&HarnessError::Session(failure));
}
}
fn reserve(&self) -> Result<Vec<Reservation>> {
let mut reservations: Vec<Reservation> = Vec::new();
let mut abandoned: Vec<TaskId> = Vec::new();
let result = self.commit(|tx| {
let marks: Vec<TaskRecord> = {
let st = self.st();
if !st.enabled || st.closing {
return Ok(());
}
st.abandoned
.iter()
.filter_map(|id| st.live.get(id))
.filter(|record| !record.abort_requested)
.map(|record| with_abort_mark(record, Some(AbortReason::Restart)))
.collect()
};
for mark in marks {
tx.set_task(mark)?;
}
{
let st = self.st();
if !st.abandoned.is_empty() {
abandoned = st.abandoned.iter().copied().collect();
return Ok(());
}
}
self.load_scopes(tx, false)?;
let (candidates, owned) = {
let st = self.st();
(st.live.values().cloned().collect::<Vec<_>>(), st.owned_live(None))
};
let mut snapshot: Option<Arc<RegistrySnapshot>> = None;
for record in candidates {
let mode = if record.abort_requested { Mode::Abort } else { Mode::Run };
{
let mut st = self.st();
if st.invocations.contains_key(&record.id) || !st.waiting_on(&record, &owned).is_empty() {
continue;
}
if record.state.status() == TaskStatus::Completing {
continue;
}
if mode == Mode::Run && !record.background && st.below_cancelled(record_parent(&record)) {
st.cascade_pending = true;
st.schedule_reconcile();
continue;
}
}
let snapshot = snapshot.get_or_insert_with(|| self.registry.snapshot()).clone();
let (task, resolved) = match self.resolve(&record, &snapshot) {
Resolution::Blocked(reason) => {
if mode == Mode::Abort && record.abort_reason.is_none() {
let outcome = SchedulerOutcome::Orphaned {
reason: reason.as_str().into(),
};
self.terminate(tx, &record, &outcome)?;
}
continue;
}
Resolution::Ready { task, record } => (task, *record),
};
if resolved != record || record.state.status() != TaskStatus::Running {
let checkpoint = checkpoint_of(&resolved).cloned().unwrap_or(Value::Null);
tx.set_task(with_state(&resolved, TaskState::Running { checkpoint }))?;
}
let invocation = self.create_invocation(&record, mode);
reservations.push(Reservation {
invocation,
task,
snapshot,
});
}
Ok(())
});
if let Err(error) = result {
let mut st = self.st();
for reservation in &reservations {
let invocation = &reservation.invocation;
if st
.invocations
.get(&invocation.task_id)
.is_some_and(|current| Arc::ptr_eq(current, invocation))
{
st.invocations.remove(&invocation.task_id);
}
invocation.done.abort();
}
return Err(error);
}
let mut st = self.st();
for id in &abandoned {
st.abandoned.shift_remove(id);
}
if !abandoned.is_empty() {
st.dirty = true;
}
Ok(reservations)
}
fn resolve(&self, record: &TaskRecord, snapshot: &RegistrySnapshot) -> Resolution {
let fit = self.st().fit(record, snapshot.task(&record.kind));
let (task, migrates) = match fit {
Fit::Blocked { reason, .. } => return Resolution::Blocked(reason),
Fit::Ready { task, migrates } => (task, migrates),
};
if !migrates {
return Resolution::Ready {
task,
record: Box::new(record.clone()),
};
}
let migrated = match task.migration() {
None => Err(missing_migration(record, &task)),
Some(migrate) => {
let checkpoint = checkpoint_of(record).cloned().unwrap_or(Value::Null);
contained(|| migrate(&record.input, &checkpoint, record.version))
}
};
match migrated {
Ok((input, checkpoint)) => {
let state = match &record.state {
TaskState::Waiting { on, policy, .. } => TaskState::Waiting {
checkpoint,
on: on.clone(),
policy: *policy,
},
TaskState::Running { .. } => TaskState::Running { checkpoint },
_ => TaskState::Pending { checkpoint },
};
Resolution::Ready {
record: Box::new(TaskRecord {
version: task.version(),
input,
state,
..record.clone()
}),
task,
}
}
Err(error) => {
self.st().failed_migrations.insert(record.id, (task, error.clone()));
(self.report)(&error);
Resolution::Blocked(BlockedReason::MigrationFailed)
}
}
}
pub(crate) fn inspect(
&self,
tx: &mut Tx<'_>,
snapshot: &RegistrySnapshot,
) -> Result<(Scheduling, Vec<TaskInspection>)> {
self.load_scopes(tx, false)?;
let st = self.st();
let owned = st.owned_live(None);
let tasks = st
.live
.values()
.map(|record| TaskInspection {
record: record.clone(),
state: self.inspect_task(&st, record, snapshot, &owned),
})
.collect();
let scheduling = if st.closing {
Scheduling::Closing
} else if st.enabled {
Scheduling::Running
} else {
Scheduling::Paused
};
Ok((scheduling, tasks))
}
fn inspect_task(
&self,
st: &State,
record: &TaskRecord,
snapshot: &RegistrySnapshot,
owned: &HashMap<TaskId, Vec<TaskId>>,
) -> TaskInspectionState {
if st.invocations.contains_key(&record.id) {
return TaskInspectionState::Running;
}
if record.state.status() == TaskStatus::Completing {
return TaskInspectionState::Completing;
}
let on = st.waiting_on(record, owned);
if !on.is_empty() {
return TaskInspectionState::Waiting { on };
}
match st.fit(record, snapshot.task(&record.kind)) {
Fit::Blocked { reason, error } => TaskInspectionState::Blocked { reason, error },
Fit::Ready { task, migrates } => {
if migrates && task.migration().is_none() {
TaskInspectionState::Blocked {
reason: BlockedReason::MigrationFailed,
error: Some(missing_migration(record, &task)),
}
} else {
TaskInspectionState::Ready { migrates }
}
}
}
}
fn create_invocation(&self, record: &TaskRecord, mode: Mode) -> Arc<Invocation> {
let invocation = Arc::new(Invocation {
task_id: record.id,
conversation_id: record.conversation_id,
mode,
controller: Cancel::new(),
ended: AtomicBool::new(false),
done: Cancel::new(),
});
self.st().invocations.insert(record.id, invocation.clone());
invocation
}
fn start(&self, reservation: Reservation) {
let scheduler = self.arc();
let name = format!("durable-task-{}", reservation.invocation.task_id.0);
let spawned = std::thread::Builder::new().name(name).spawn(move || {
let invocation = reservation.invocation.clone();
let result = match invocation.mode {
Mode::Run => scheduler.run(reservation),
Mode::Abort => scheduler.run_abort(reservation),
};
if let Err(error) = result {
(scheduler.report)(&error);
}
scheduler.end(&invocation);
invocation.done.abort();
scheduler.kick();
});
if let Err(error) = spawned {
(self.report)(&HarnessError::Error(error.to_string()));
}
}
fn run(&self, reservation: Reservation) -> Result<()> {
let invocation = reservation.invocation;
let phase = Arc::new(Mutex::new(PhaseState {
task: reservation.task,
snapshot: reservation.snapshot,
reported: None,
}));
let runtime = TaskRuntime::new(self.arc(), invocation.clone(), phase.clone());
let mut previous: Option<PhaseResult> = None;
loop {
let current = self.step(&invocation, |tx, current| {
self.decide(tx, current, previous.as_ref(), &phase)
});
let Some(current) = current else { return Ok(()) };
if self.is_closing() {
return Ok(());
}
let checkpoint = checkpoint_of(¤t).cloned().unwrap_or(Value::Null);
let task = lock(&phase).task.clone();
let name = phase_of(¤t).unwrap_or_default().to_owned();
let result = match task.phase(&name) {
Some(handler) => contained(|| handler(¤t, &runtime)),
None => Err(HarnessError::Error(format!(
"Task {} has no phase {}",
current.kind, name
))),
};
previous = Some(PhaseResult {
checkpoint,
failure: result.err(),
});
}
}
fn decide(
&self,
tx: &mut Tx<'_>,
current: &TaskRecord,
previous: Option<&PhaseResult>,
phase: &Mutex<PhaseState>,
) -> Result<Decision> {
if current.abort_requested {
return Ok(Decision::End);
}
{
let mut st = self.st();
if !current.background && st.below_cancelled(record_parent(current)) {
st.cascade_pending = true;
st.schedule_reconcile();
drop(st);
self.wake.notify_all();
return Ok(Decision::End);
}
}
let Some(previous) = previous else {
return Ok(Decision::Continue);
};
if let Some(failure) = &previous.failure {
return Ok(Decision::Fault(failure.clone()));
}
let checkpoint = checkpoint_of(current).cloned().unwrap_or(Value::Null);
if json_equal(&checkpoint, &previous.checkpoint) {
let name = previous
.checkpoint
.get("phase")
.and_then(Value::as_str)
.unwrap_or_default();
return Ok(Decision::Fault(HarnessError::Error(format!(
"Task {} phase {} returned without durable progress",
current.kind, name
))));
}
let snapshot = self.registry.snapshot();
let next = snapshot.task(¤t.kind).cloned();
let report = {
let mut phase = lock(phase);
phase.snapshot = snapshot;
let replaced = match &next {
Some(next) => !next.same(&phase.task),
None => true,
};
if !replaced {
None
} else if next.as_ref().is_some_and(|next| can_reserve(next, current)) {
drop(phase);
tx.set_task(with_state(current, TaskState::Pending { checkpoint }))?;
return Ok(Decision::End);
} else {
let already = phase.reported.as_ref().is_some_and(|reported| match (reported, &next) {
(None, None) => true,
(Some(reported), Some(next)) => reported.same(next),
_ => false,
});
if already {
None
} else {
phase.reported = Some(next.clone());
Some(HarnessError::Caused {
message: format!(
"Task {} keeps running under its old {} definition",
current.id.0, current.kind
),
cause: if next.is_none() {
"missing_task"
} else {
"incompatible_task"
}
.into(),
})
}
}
};
if let Some(error) = report {
(self.report)(&error);
}
Ok(Decision::Continue)
}
fn run_abort(&self, reservation: Reservation) -> Result<()> {
let invocation = reservation.invocation;
let current = self.st().live.get(&invocation.task_id).cloned();
let Some(current) = current else { return Ok(()) };
if self.is_closing() {
return Ok(());
}
let phase = Arc::new(Mutex::new(PhaseState {
task: reservation.task.clone(),
snapshot: reservation.snapshot,
reported: None,
}));
let runtime = TaskRuntime::new(self.arc(), invocation.clone(), phase);
let handler = reservation.task.abort_handler().clone();
let failure = contained(|| handler(¤t, &runtime)).err();
let message = format!(
"Abort handler of task {} returned without a terminal outcome",
invocation.task_id.0
);
let fault = failure.unwrap_or(HarnessError::Error(message));
self.step(&invocation, |_, _| Ok(Decision::Fault(fault)));
Ok(())
}
fn step(
&self,
invocation: &Arc<Invocation>,
decide: impl FnOnce(&mut Tx<'_>, &TaskRecord) -> Result<Decision>,
) -> Option<TaskRecord> {
let result = self.commit(|tx| {
let current = {
let st = self.st();
match st.live.get(&invocation.task_id) {
Some(found) if found.state.status() == TaskStatus::Running && !st.closing => Some(found.clone()),
_ => None,
}
};
let decision = match ¤t {
Some(current) => decide(tx, current)?,
None => Decision::End,
};
if let Decision::Continue = decision {
return Ok(current);
}
self.end(invocation);
if let (Decision::Fault(error), Some(current)) = (decision, ¤t) {
let outcome = SchedulerOutcome::Faulted {
message: error.message(),
};
self.terminate(tx, current, &outcome)?;
}
Ok(None)
});
match result {
Ok(current) => current,
Err(error) => {
self.end(invocation);
self.fail_session(error);
None
}
}
}
fn terminate(&self, tx: &mut Tx<'_>, record: &TaskRecord, outcome: &SchedulerOutcome) -> Result<()> {
self.load_scopes(tx, false)?;
let overlay = overlay_of(tx);
let held = self.st().owned_live(Some(&overlay)).contains_key(&record.id);
if held {
tx.set_task(with_state(
record,
TaskState::Completing {
outcome: outcome.outcome(),
},
))?;
return Ok(());
}
tx.set_task(with_state(
record,
TaskState::Terminal {
outcome: outcome.outcome(),
},
))?;
(self.settle_outcome)(tx, record, outcome)
}
pub(crate) fn commit_state(
&self,
tx: &mut Tx<'_>,
invocation: &Invocation,
current: &TaskRecord,
next: NextState,
) -> Result<()> {
let state = match next {
NextState::Running { checkpoint } => TaskState::Running { checkpoint },
NextState::Waiting { checkpoint, on, policy } => {
self.validate_wait(tx, invocation, current, &on, policy)?;
TaskState::Waiting { checkpoint, on, policy }
}
NextState::Terminal { outcome } => {
let overlay = overlay_of(tx);
self.load_scopes(tx, false)?;
for record in overlay.tasks.values() {
self.load_chain(tx, record_parent(record), Some(&overlay))?;
}
if self.st().owned_live(Some(&overlay)).contains_key(¤t.id) {
tx.set_task(with_state(current, TaskState::Completing { outcome }))?;
return Ok(());
}
TaskState::Terminal { outcome }
}
};
tx.set_task(with_state(current, state))?;
Ok(())
}
fn validate_wait(
&self,
tx: &mut Tx<'_>,
invocation: &Invocation,
current: &TaskRecord,
on: &[TaskId],
policy: JoinPolicy,
) -> Result<()> {
if invocation.mode == Mode::Abort {
return Err(HarnessError::Error(format!(
"Abort handler of task {} cannot wait",
current.id.0
)));
}
let overlay = overlay_of(tx);
self.load_chain(tx, record_parent(current), None)?;
let owners: HashSet<TaskId> = self
.st()
.above(record_parent(current), None)
.into_iter()
.filter_map(|step| match step {
Step::Task(id, _) => Some(id),
_ => None,
})
.collect();
for id in on {
if *id == current.id || owners.contains(id) {
return Err(HarnessError::Error(format!(
"Task {} cannot wait on itself or its owner {}",
current.id.0, id.0
)));
}
let known = overlay
.tasks
.get(id)
.cloned()
.or_else(|| self.st().live.get(id).cloned());
let member = match known {
Some(member) => Some(member),
None => tx.committed(|storage| storage.task(*id))?,
};
let Some(member) = member else {
return Err(HarnessError::Error(format!("Task {} does not exist", id.0)));
};
if policy == JoinPolicy::FailFast && member.owner != Some(current.id) {
return Err(HarnessError::Error(format!(
"Task {} can wait failFast only on tasks it owns; {} is not one",
current.id.0, id.0
)));
}
}
Ok(())
}
pub(crate) fn end(&self, invocation: &Arc<Invocation>) {
if invocation.ended.swap(true, Ordering::SeqCst) {
return;
}
{
let mut st = self.st();
if st
.invocations
.get(&invocation.task_id)
.is_some_and(|current| Arc::ptr_eq(current, invocation))
{
st.invocations.remove(&invocation.task_id);
}
}
invocation.controller.cancel(invocation.ended_error());
}
pub(crate) fn live_record(&self, id: TaskId) -> Option<TaskRecord> {
self.st().live.get(&id).cloned()
}
pub(crate) fn registry(&self) -> &Arc<dyn RegistryReader> {
&self.registry
}
}