#[cfg(test)]
use std::collections::HashMap;
use std::collections::{BTreeMap, BTreeSet};
#[cfg(test)]
use std::num::{NonZeroU64, NonZeroUsize};
#[cfg(test)]
use std::sync::RwLock;
#[cfg(test)]
use std::sync::atomic::Ordering;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering as TaskServiceOrdering};
use std::sync::{Arc, Mutex};
use std::time::{Duration as StdDuration, Instant};
use std::future::Future;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::pin::Pin;
#[cfg(test)]
use asupersync::Budget;
#[cfg(test)]
use asupersync::CancelKind;
use asupersync::Cx;
use asupersync::channel::mpsc::{self, Receiver, Sender};
#[cfg(test)]
use asupersync::runtime::{RuntimeBuilder, RuntimeHandle};
use base64::Engine as _;
use fastmcp_core::crypto::Sha256Digest;
#[cfg(test)]
use fastmcp_core::logging::{debug, info, targets, warn};
use fastmcp_core::{McpContext, McpError, McpResult, draw_security_identifier};
use fastmcp_protocol::tasks_extension::TaskStatusNotificationParams as FinalTaskStatusNotificationParams;
use fastmcp_protocol::{
CreateTaskResult, FINAL_PROTOCOL_VERSION, FinalCancelTaskParams, FinalCancelTaskResult,
FinalGetTaskParams, FinalGetTaskResult, FinalTaskCallToolResult, FinalTaskError, FinalTaskId,
FinalTaskStatus, Task as FinalTask, TaskBase as FinalTaskBase,
TaskDuration as FinalTaskDuration, TaskInputLedger as FinalTaskInputLedger,
TaskInputRequests as FinalTaskInputRequests, TaskInputResponses as FinalTaskInputResponses,
TaskRequestMeta as FinalTaskRequestMeta, TaskStatusNotification as FinalTaskStatusNotification,
TaskTimestamp as FinalTaskTimestamp, UpdateTaskParams, UpdateTaskResult,
};
#[cfg(test)]
use fastmcp_protocol::{
JsonRpcRequest, TaskId, TaskInfo, TaskResult, TaskStatus, TaskStatusNotificationParams,
};
#[cfg(test)]
pub type TaskNotificationSender = Arc<dyn Fn(JsonRpcRequest) + Send + Sync>;
#[cfg(test)]
pub type TaskHandler = Box<dyn Fn(&Cx, serde_json::Value) -> TaskFuture + Send + Sync + 'static>;
#[cfg(test)]
pub type TaskFuture = std::pin::Pin<
Box<dyn std::future::Future<Output = McpResult<serde_json::Value>> + Send + 'static>,
>;
#[cfg(test)]
struct TaskState {
info: TaskInfo,
cancel_requested: bool,
result: Option<TaskResult>,
cx: Cx,
}
#[cfg(test)]
fn can_transition(from: TaskStatus, to: TaskStatus) -> bool {
matches!(
(from, to),
(
TaskStatus::Pending,
TaskStatus::Running | TaskStatus::Failed | TaskStatus::Cancelled
) | (
TaskStatus::Running,
TaskStatus::Completed | TaskStatus::Failed | TaskStatus::Cancelled
)
)
}
#[cfg(test)]
fn transition_state(state: &mut TaskState, to: TaskStatus) -> bool {
let from = state.info.status;
if from == to {
return true;
}
if !can_transition(from, to) {
warn!(
target: targets::SERVER,
"task {} invalid transition {:?} -> {:?}",
state.info.id,
from,
to
);
return false;
}
state.info.status = to;
let now = chrono::Utc::now().to_rfc3339();
match to {
TaskStatus::Running => {
state.info.started_at = Some(now.clone());
}
TaskStatus::Completed | TaskStatus::Failed | TaskStatus::Cancelled => {
state.info.completed_at = Some(now.clone());
}
TaskStatus::Pending => {}
}
info!(
target: targets::SERVER,
"task {} status {:?} -> {:?} at {}",
state.info.id,
from,
to,
now
);
true
}
#[cfg(test)]
fn mark_task_failed_snapshot(
tasks: &Arc<RwLock<HashMap<TaskId, TaskState>>>,
task_id: &TaskId,
error_msg: String,
lock_context: &'static str,
) -> Option<TaskStatusSnapshot> {
let mut tasks_guard = tasks.write().unwrap_or_else(|poisoned| {
warn!(
target: targets::SERVER,
"tasks lock poisoned in {}, recovering",
lock_context
);
poisoned.into_inner()
});
let state = tasks_guard.get_mut(task_id)?;
if state.cancel_requested || !transition_state(state, TaskStatus::Failed) {
return None;
}
state.info.error = Some(error_msg.clone());
state.result = Some(TaskResult {
id: task_id.clone(),
success: false,
data: None,
error: Some(error_msg),
});
Some(TaskStatusSnapshot::from(state))
}
#[cfg(test)]
fn build_runtime_handle() -> Option<RuntimeHandle> {
match RuntimeBuilder::multi_thread().build() {
Ok(runtime) => Some(runtime.handle()),
Err(multi_err) => {
warn!(
target: targets::SERVER,
"failed to initialize multi-thread runtime for tasks: {}; attempting current-thread fallback",
multi_err
);
match RuntimeBuilder::current_thread().build() {
Ok(runtime) => Some(runtime.handle()),
Err(single_err) => {
warn!(
target: targets::SERVER,
"failed to initialize current-thread runtime fallback for tasks: {}",
single_err
);
None
}
}
}
}
}
#[cfg(test)]
pub struct TaskManager {
tasks: Arc<RwLock<HashMap<TaskId, TaskState>>>,
handlers: Arc<RwLock<HashMap<String, TaskHandler>>>,
task_counter: AtomicU64,
list_changed_notifications: bool,
runtime: Option<RuntimeHandle>,
auto_execute: bool,
notification_sender: Arc<RwLock<Option<TaskNotificationSender>>>,
}
#[cfg(test)]
impl TaskManager {
#[must_use]
pub fn new() -> Self {
let runtime = build_runtime_handle();
if runtime.is_none() {
warn!(
target: targets::SERVER,
"TaskManager runtime unavailable; auto-executed tasks will fail until runtime becomes available"
);
}
Self {
tasks: Arc::new(RwLock::new(HashMap::new())),
handlers: Arc::new(RwLock::new(HashMap::new())),
task_counter: AtomicU64::new(0),
list_changed_notifications: false,
runtime,
auto_execute: true,
notification_sender: Arc::new(RwLock::new(None)),
}
}
#[must_use]
pub fn with_list_changed_notifications() -> Self {
Self {
list_changed_notifications: true,
..Self::new()
}
}
#[must_use]
pub fn new_for_testing() -> Self {
let mut manager = Self::new();
manager.auto_execute = false;
manager
}
#[must_use]
pub fn into_shared(self) -> SharedTaskManager {
Arc::new(self)
}
#[must_use]
pub fn has_list_changed_notifications(&self) -> bool {
self.list_changed_notifications
}
pub fn set_notification_sender(&self, sender: TaskNotificationSender) {
let mut guard = self.notification_sender.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "notification sender lock poisoned, recovering");
poisoned.into_inner()
});
*guard = Some(sender);
}
pub fn register_handler<F, Fut>(&self, task_type: impl Into<String>, handler: F)
where
F: Fn(&Cx, serde_json::Value) -> Fut + Send + Sync + 'static,
Fut: std::future::Future<Output = McpResult<serde_json::Value>> + Send + 'static,
{
let task_type = task_type.into();
let boxed_handler: TaskHandler = Box::new(move |cx, params| Box::pin(handler(cx, params)));
let mut handlers = self.handlers.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "handlers lock poisoned, recovering");
poisoned.into_inner()
});
handlers.insert(task_type, boxed_handler);
}
pub fn submit(
&self,
cx: &Cx,
task_type: impl Into<String>,
params: Option<serde_json::Value>,
) -> McpResult<TaskId> {
let task_type = task_type.into();
{
let handlers = self.handlers.read().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "handlers lock poisoned, recovering");
poisoned.into_inner()
});
if !handlers.contains_key(&task_type) {
return Err(McpError::invalid_params(format!(
"Unknown task type: {task_type}"
)));
}
}
let counter = self.task_counter.fetch_add(1, Ordering::SeqCst);
let task_id = TaskId::from_string(format!("task-{counter:08x}"));
let now = chrono::Utc::now().to_rfc3339();
let task_cx = cx.clone();
let info = TaskInfo {
id: task_id.clone(),
task_type: task_type.clone(),
status: TaskStatus::Pending,
progress: None,
message: None,
created_at: now,
started_at: None,
completed_at: None,
error: None,
};
let info_snapshot = info.clone();
let state = TaskState {
info,
cancel_requested: false,
result: None,
cx: task_cx.clone(),
};
{
let mut tasks = self.tasks.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned, recovering");
poisoned.into_inner()
});
tasks.insert(task_id.clone(), state);
}
self.notify_status(info_snapshot, None);
if self.auto_execute {
let params = params.unwrap_or_else(|| serde_json::json!({}));
self.spawn_task(task_id.clone(), task_type, task_cx, params);
}
Ok(task_id)
}
#[allow(clippy::too_many_lines)]
fn spawn_task(
&self,
task_id: TaskId,
task_type: String,
task_cx: Cx,
params: serde_json::Value,
) {
let Some(runtime) = self.runtime.clone() else {
let failure_snapshot = mark_task_failed_snapshot(
&self.tasks,
&task_id,
"Task runtime unavailable".to_string(),
"spawn_task runtime unavailable",
);
self.notify_snapshot(failure_snapshot);
return;
};
let tasks = Arc::clone(&self.tasks);
let handlers = Arc::clone(&self.handlers);
let notification_sender = Arc::clone(&self.notification_sender);
let scheduled_task_id = task_id.clone();
let scheduling = runtime.try_spawn(async move {
let running_snapshot = {
let mut tasks_guard = tasks.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in spawn_task, recovering");
poisoned.into_inner()
});
match tasks_guard.get_mut(&task_id) {
Some(state) => {
if state.cancel_requested || !transition_state(state, TaskStatus::Running) {
None
} else {
Some(TaskStatusSnapshot::from(state))
}
}
None => None,
}
};
let should_start = running_snapshot.is_some();
notify_snapshot(¬ification_sender, running_snapshot);
if !should_start {
return;
}
let task_future = {
let handlers_guard = handlers.read().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "handlers lock poisoned in spawn_task, recovering");
poisoned.into_inner()
});
let Some(handler) = handlers_guard.get(&task_type) else {
let failure_snapshot = mark_task_failed_snapshot(
&tasks,
&task_id,
format!("Unknown task type: {task_type}"),
"spawn_task failure",
);
notify_snapshot(¬ification_sender, failure_snapshot);
return;
};
(handler)(&task_cx, params)
};
let result = task_future.await;
let completion_snapshot = {
let mut tasks_guard = tasks.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in spawn_task completion, recovering");
poisoned.into_inner()
});
match tasks_guard.get_mut(&task_id) {
Some(state) => {
if state.cancel_requested {
None
} else {
let mut snapshot = None;
match result {
Ok(data) => {
if transition_state(state, TaskStatus::Completed) {
state.info.progress = Some(1.0);
state.result = Some(TaskResult {
id: task_id.clone(),
success: true,
data: Some(data),
error: None,
});
snapshot = Some(TaskStatusSnapshot::from(state));
}
}
Err(err) => {
let error_msg = err.message;
if transition_state(state, TaskStatus::Failed) {
state.info.error = Some(error_msg.clone());
state.result = Some(TaskResult {
id: task_id.clone(),
success: false,
data: None,
error: Some(error_msg),
});
snapshot = Some(TaskStatusSnapshot::from(state));
}
}
}
snapshot
}
}
None => None,
}
};
notify_snapshot(¬ification_sender, completion_snapshot);
});
if let Err(err) = scheduling {
warn!(
target: targets::SERVER,
"failed to schedule task {}: {}",
scheduled_task_id,
err
);
let failure_snapshot = mark_task_failed_snapshot(
&self.tasks,
&scheduled_task_id,
format!("Failed to schedule task: {err}"),
"spawn_task scheduling",
);
self.notify_snapshot(failure_snapshot);
}
}
pub fn start_task(&self, task_id: &TaskId) -> McpResult<()> {
let snapshot = {
let mut tasks = self.tasks.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in start_task, recovering");
poisoned.into_inner()
});
let state = tasks
.get_mut(task_id)
.ok_or_else(|| McpError::invalid_params(format!("Task not found: {task_id}")))?;
if state.info.status != TaskStatus::Pending {
return Err(McpError::invalid_params(format!(
"Task {task_id} is not pending"
)));
}
if !transition_state(state, TaskStatus::Running) {
return Err(McpError::invalid_params(format!(
"Task {task_id} cannot transition to running"
)));
}
Some(TaskStatusSnapshot::from(state))
};
self.notify_snapshot(snapshot);
Ok(())
}
pub fn update_progress(&self, task_id: &TaskId, progress: f64, message: Option<String>) {
let snapshot = {
let mut tasks = self.tasks.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in update_progress, recovering");
poisoned.into_inner()
});
if let Some(state) = tasks.get_mut(task_id) {
if state.info.status != TaskStatus::Running {
debug!(
target: targets::SERVER,
"task {} progress update ignored in state {:?}",
task_id,
state.info.status
);
return;
}
state.info.progress = Some(progress.clamp(0.0, 1.0));
state.info.message = message;
Some(TaskStatusSnapshot::from(state))
} else {
None
}
};
self.notify_snapshot(snapshot);
}
pub fn complete_task(&self, task_id: &TaskId, data: serde_json::Value) {
let snapshot = {
let mut tasks = self.tasks.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in complete_task, recovering");
poisoned.into_inner()
});
if let Some(state) = tasks.get_mut(task_id) {
if !transition_state(state, TaskStatus::Completed) {
return;
}
state.info.progress = Some(1.0);
state.result = Some(TaskResult {
id: task_id.clone(),
success: true,
data: Some(data),
error: None,
});
Some(TaskStatusSnapshot::from(state))
} else {
None
}
};
self.notify_snapshot(snapshot);
}
pub fn fail_task(&self, task_id: &TaskId, error: impl Into<String>) {
let error = error.into();
let snapshot = {
let mut tasks = self.tasks.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in fail_task, recovering");
poisoned.into_inner()
});
if let Some(state) = tasks.get_mut(task_id) {
if !transition_state(state, TaskStatus::Failed) {
return;
}
state.info.error = Some(error.clone());
state.result = Some(TaskResult {
id: task_id.clone(),
success: false,
data: None,
error: Some(error),
});
Some(TaskStatusSnapshot::from(state))
} else {
None
}
};
self.notify_snapshot(snapshot);
}
#[must_use]
pub fn get_info(&self, task_id: &TaskId) -> Option<TaskInfo> {
let tasks = self.tasks.read().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in get_info, recovering");
poisoned.into_inner()
});
tasks.get(task_id).map(|s| s.info.clone())
}
#[must_use]
pub fn get_result(&self, task_id: &TaskId) -> Option<TaskResult> {
let tasks = self.tasks.read().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in get_result, recovering");
poisoned.into_inner()
});
tasks.get(task_id).and_then(|s| s.result.clone())
}
#[must_use]
pub fn list_tasks(&self, status_filter: Option<TaskStatus>) -> Vec<TaskInfo> {
let tasks = self.tasks.read().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in list_tasks, recovering");
poisoned.into_inner()
});
tasks
.values()
.filter(|s| status_filter.is_none_or(|f| s.info.status == f))
.map(|s| s.info.clone())
.collect()
}
pub fn cancel(&self, task_id: &TaskId, reason: Option<String>) -> McpResult<TaskInfo> {
let snapshot = {
let mut tasks = self.tasks.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in cancel, recovering");
poisoned.into_inner()
});
let state = tasks
.get_mut(task_id)
.ok_or_else(|| McpError::invalid_params(format!("Task not found: {task_id}")))?;
if state.info.status.is_terminal() {
return Err(McpError::invalid_params(format!(
"Task {task_id} is already in terminal state: {:?}",
state.info.status
)));
}
if !transition_state(state, TaskStatus::Cancelled) {
return Err(McpError::invalid_params(format!(
"Task {task_id} cannot be cancelled from {:?}",
state.info.status
)));
}
state.cancel_requested = true;
state.cx.cancel_with(CancelKind::User, None);
if !state.cx.is_cancel_requested() {
warn!(
target: targets::SERVER,
"task {} cancel signal not observed on context",
task_id
);
}
let error_msg = reason.unwrap_or_else(|| "Cancelled by request".to_string());
state.info.error = Some(error_msg.clone());
state.result = Some(TaskResult {
id: task_id.clone(),
success: false,
data: None,
error: Some(error_msg),
});
let snapshot = TaskStatusSnapshot::from(state);
(snapshot, state.info.clone())
};
let (snapshot, info) = snapshot;
self.notify_snapshot(Some(snapshot));
Ok(info)
}
#[must_use]
pub fn is_cancel_requested(&self, task_id: &TaskId) -> bool {
let tasks = self.tasks.read().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in is_cancel_requested, recovering");
poisoned.into_inner()
});
tasks.get(task_id).is_some_and(|s| s.cancel_requested)
}
#[must_use]
pub fn active_count(&self) -> usize {
let tasks = self.tasks.read().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in active_count, recovering");
poisoned.into_inner()
});
tasks.values().filter(|s| s.info.status.is_active()).count()
}
#[must_use]
pub fn total_count(&self) -> usize {
let tasks = self.tasks.read().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in total_count, recovering");
poisoned.into_inner()
});
tasks.len()
}
pub fn cleanup_completed(&self, max_age: std::time::Duration) {
let cutoff = chrono::Utc::now() - chrono::Duration::from_std(max_age).unwrap_or_default();
let mut tasks = self.tasks.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in cleanup_completed, recovering");
poisoned.into_inner()
});
tasks.retain(|_, state| {
if state.info.status.is_active() {
return true;
}
if let Some(ref completed) = state.info.completed_at {
if let Ok(parsed) = chrono::DateTime::parse_from_rfc3339(completed) {
return parsed.with_timezone(&chrono::Utc) > cutoff;
}
return true;
}
true
});
}
fn notify_snapshot(&self, snapshot: Option<TaskStatusSnapshot>) {
if let Some(snapshot) = snapshot {
self.notify_status(snapshot.info, snapshot.result);
}
}
fn notify_status(&self, info: TaskInfo, result: Option<TaskResult>) {
let sender = {
let guard = self.notification_sender.read().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "notification sender lock poisoned in notify_status, recovering");
poisoned.into_inner()
});
guard.clone()
};
let Some(sender) = sender else {
return;
};
let params = TaskStatusNotificationParams {
id: info.id.clone(),
status: info.status,
progress: info.progress,
message: info.message.clone(),
error: info.error.clone(),
result,
};
let payload = match serde_json::to_value(params) {
Ok(value) => value,
Err(err) => {
warn!(
target: targets::SERVER,
"failed to serialize task status notification: {}",
err
);
return;
}
};
sender(JsonRpcRequest::notification(
"notifications/tasks/status",
Some(payload),
));
}
}
#[cfg(test)]
#[derive(Debug, Clone)]
struct TaskStatusSnapshot {
info: TaskInfo,
result: Option<TaskResult>,
}
#[cfg(test)]
impl TaskStatusSnapshot {
fn from(state: &TaskState) -> Self {
Self {
info: state.info.clone(),
result: state.result.clone(),
}
}
}
#[cfg(test)]
fn notify_snapshot(
sender: &Arc<RwLock<Option<TaskNotificationSender>>>,
snapshot: Option<TaskStatusSnapshot>,
) {
let Some(snapshot) = snapshot else {
return;
};
let sender = {
let guard = sender.read().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "notification sender lock poisoned in notify_snapshot, recovering");
poisoned.into_inner()
});
guard.clone()
};
let Some(sender) = sender else {
return;
};
let params = TaskStatusNotificationParams {
id: snapshot.info.id.clone(),
status: snapshot.info.status,
progress: snapshot.info.progress,
message: snapshot.info.message.clone(),
error: snapshot.info.error.clone(),
result: snapshot.result,
};
let payload = match serde_json::to_value(params) {
Ok(value) => value,
Err(err) => {
warn!(
target: targets::SERVER,
"failed to serialize task status notification: {}",
err
);
return;
}
};
sender(JsonRpcRequest::notification(
"notifications/tasks/status",
Some(payload),
));
}
#[cfg(test)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TaskStorageKind {
ProcessLocal,
}
#[cfg(test)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub(crate) enum OfficialTaskStatus {
#[serde(rename = "working")]
Working,
#[serde(rename = "input_required")]
InputRequired,
#[serde(rename = "completed")]
Completed,
#[serde(rename = "failed")]
Failed,
#[serde(rename = "cancelled")]
Cancelled,
}
#[cfg(test)]
impl OfficialTaskStatus {
#[must_use]
fn is_terminal(self) -> bool {
matches!(self, Self::Completed | Self::Failed | Self::Cancelled)
}
}
#[cfg(test)]
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub(crate) struct OfficialTaskInputRequest {
pub method: OfficialTaskInputMethod,
pub params: serde_json::Value,
}
#[cfg(test)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
pub(crate) enum OfficialTaskInputMethod {
#[serde(rename = "elicitation/create")]
ElicitationCreate,
}
#[cfg(test)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct OfficialTaskLifecycleConfig {
ttl_ms: NonZeroU64,
poll_interval_ms: Option<NonZeroU64>,
max_tasks: NonZeroUsize,
}
#[cfg(test)]
impl OfficialTaskLifecycleConfig {
pub(crate) fn new(
ttl_ms: u64,
poll_interval_ms: Option<u64>,
max_tasks: usize,
) -> McpResult<Self> {
let ttl_ms = NonZeroU64::new(ttl_ms)
.ok_or_else(|| McpError::invalid_params("Task TTL must be a positive integer"))?;
let poll_interval_ms = match poll_interval_ms {
Some(poll_interval_ms) => Some(NonZeroU64::new(poll_interval_ms).ok_or_else(|| {
McpError::invalid_params("Task poll interval must be a positive integer")
})?),
None => None,
};
let max_tasks = NonZeroUsize::new(max_tasks)
.ok_or_else(|| McpError::invalid_params("Task capacity must be positive"))?;
Ok(Self {
ttl_ms,
poll_interval_ms,
max_tasks,
})
}
}
#[cfg(test)]
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub(crate) struct OfficialTaskSnapshot {
#[serde(rename = "taskId")]
pub task_id: TaskId,
pub status: OfficialTaskStatus,
#[serde(rename = "statusMessage", skip_serializing_if = "Option::is_none")]
pub status_message: Option<String>,
#[serde(rename = "createdAt")]
pub created_at: String,
#[serde(rename = "lastUpdatedAt")]
pub last_updated_at: String,
#[serde(rename = "ttlMs")]
pub ttl_ms: u64,
#[serde(rename = "pollIntervalMs", skip_serializing_if = "Option::is_none")]
pub poll_interval_ms: Option<u64>,
#[serde(rename = "inputRequests", skip_serializing_if = "Option::is_none")]
pub input_requests: Option<BTreeMap<String, OfficialTaskInputRequest>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub result: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<serde_json::Value>,
}
#[cfg(test)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum OfficialTaskInputUpdate {
Applied,
Ignored,
}
#[cfg(test)]
struct OfficialTaskRecord {
snapshot: OfficialTaskSnapshot,
expires_at: Instant,
update_revision: u64,
issued_input_keys: BTreeSet<String>,
cancellation_requested: bool,
}
#[cfg(test)]
pub(crate) struct OfficialTaskLifecycle {
config: OfficialTaskLifecycleConfig,
records: RwLock<HashMap<TaskId, OfficialTaskRecord>>,
}
#[cfg(test)]
impl OfficialTaskLifecycle {
#[must_use]
pub(crate) fn new(config: OfficialTaskLifecycleConfig) -> Self {
Self {
config,
records: RwLock::new(HashMap::new()),
}
}
#[must_use]
pub(crate) const fn storage_kind(&self) -> TaskStorageKind {
TaskStorageKind::ProcessLocal
}
pub(crate) fn create(&self, status_message: Option<String>) -> McpResult<OfficialTaskSnapshot> {
let expires_at = Instant::now()
.checked_add(StdDuration::from_millis(self.config.ttl_ms.get()))
.ok_or_else(|| {
McpError::internal_error("Task TTL exceeds process-local clock range")
})?;
let mut records = self.records.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "official task lifecycle lock poisoned in create, recovering");
poisoned.into_inner()
});
let now = Instant::now();
records.retain(|_, record| record.expires_at > now);
if records.len() >= self.config.max_tasks.get() {
return Err(McpError::internal_error(
"Process-local task lifecycle capacity is exhausted",
));
}
for _ in 0..4 {
let task_id = generate_official_task_id()?;
if records.contains_key(&task_id) {
continue;
}
let now = official_task_timestamp();
let snapshot = OfficialTaskSnapshot {
task_id: task_id.clone(),
status: OfficialTaskStatus::Working,
status_message,
created_at: now.clone(),
last_updated_at: now,
ttl_ms: self.config.ttl_ms.get(),
poll_interval_ms: self.config.poll_interval_ms.map(NonZeroU64::get),
input_requests: None,
result: None,
error: None,
};
records.insert(
task_id,
OfficialTaskRecord {
snapshot: snapshot.clone(),
expires_at,
update_revision: 0,
issued_input_keys: BTreeSet::new(),
cancellation_requested: false,
},
);
return Ok(snapshot);
}
Err(McpError::internal_error(
"Unable to allocate a unique task identifier after four secure draws",
))
}
pub(crate) fn get(&self, task_id: &TaskId) -> McpResult<OfficialTaskSnapshot> {
let mut records = self.records.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "official task lifecycle lock poisoned in get, recovering");
poisoned.into_inner()
});
Ok(official_task_record_mut(&mut records, task_id)?
.snapshot
.clone())
}
pub(crate) fn require_input(
&self,
task_id: &TaskId,
requests: BTreeMap<String, OfficialTaskInputRequest>,
status_message: Option<String>,
) -> McpResult<OfficialTaskSnapshot> {
if requests.is_empty() {
return Err(McpError::invalid_params(
"input_required tasks need at least one outstanding input request",
));
}
for (key, request) in &requests {
validate_official_task_input_request(key, request)?;
}
let mut records = self.records.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "official task lifecycle lock poisoned in require_input, recovering");
poisoned.into_inner()
});
let record = official_task_record_mut(&mut records, task_id)?;
if record.snapshot.status != OfficialTaskStatus::Working {
return Err(invalid_official_task_transition(
record.snapshot.status,
OfficialTaskStatus::InputRequired,
));
}
if requests
.keys()
.any(|key| record.issued_input_keys.contains(key))
{
return Err(McpError::invalid_params(
"Task input request keys cannot be reused",
));
}
advance_official_task(record, OfficialTaskStatus::InputRequired)?;
record.issued_input_keys.extend(requests.keys().cloned());
record.snapshot.status_message = status_message;
record.snapshot.input_requests = Some(requests);
record.snapshot.result = None;
record.snapshot.error = None;
Ok(record.snapshot.clone())
}
pub(crate) fn update_input(
&self,
task_id: &TaskId,
responses: BTreeMap<String, serde_json::Value>,
) -> McpResult<OfficialTaskInputUpdate> {
let mut records = self.records.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "official task lifecycle lock poisoned in update_input, recovering");
poisoned.into_inner()
});
let record = official_task_record_mut(&mut records, task_id)?;
let all_inputs_satisfied = {
let Some(outstanding) = record.snapshot.input_requests.as_mut() else {
return Ok(OfficialTaskInputUpdate::Ignored);
};
let matched_keys: Vec<String> = responses
.keys()
.filter(|key| outstanding.contains_key(*key))
.cloned()
.collect();
if matched_keys.is_empty() {
return Ok(OfficialTaskInputUpdate::Ignored);
}
for key in matched_keys {
outstanding.remove(&key);
}
outstanding.is_empty()
};
if all_inputs_satisfied {
advance_official_task(record, OfficialTaskStatus::Working)?;
record.snapshot.input_requests = None;
record.snapshot.status_message = None;
} else {
touch_official_task(record)?;
}
Ok(OfficialTaskInputUpdate::Applied)
}
pub(crate) fn request_cancellation(&self, task_id: &TaskId) -> McpResult<()> {
let mut records = self.records.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "official task lifecycle lock poisoned in request_cancellation, recovering");
poisoned.into_inner()
});
let record = official_task_record_mut(&mut records, task_id)?;
record.cancellation_requested = true;
Ok(())
}
#[must_use]
pub(crate) fn is_cancellation_requested(&self, task_id: &TaskId) -> bool {
let mut records = self.records.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "official task lifecycle lock poisoned in is_cancellation_requested, recovering");
poisoned.into_inner()
});
official_task_record_mut(&mut records, task_id).is_ok_and(|record| {
record.cancellation_requested && !record.snapshot.status.is_terminal()
})
}
pub(crate) fn honor_cancellation(
&self,
task_id: &TaskId,
status_message: Option<String>,
) -> McpResult<OfficialTaskSnapshot> {
let mut records = self.records.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "official task lifecycle lock poisoned in honour_cancellation, recovering");
poisoned.into_inner()
});
let record = official_task_record_mut(&mut records, task_id)?;
if !record.cancellation_requested {
return Err(McpError::invalid_params(
"Task cancellation has not been requested",
));
}
transition_to_terminal(
record,
OfficialTaskStatus::Cancelled,
status_message,
None,
None,
)
}
pub(crate) fn complete(
&self,
task_id: &TaskId,
result: serde_json::Value,
status_message: Option<String>,
) -> McpResult<OfficialTaskSnapshot> {
validate_final_tool_result(&result)?;
let mut records = self.records.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "official task lifecycle lock poisoned in complete, recovering");
poisoned.into_inner()
});
let record = official_task_record_mut(&mut records, task_id)?;
transition_to_terminal(
record,
OfficialTaskStatus::Completed,
status_message,
Some(result),
None,
)
}
pub(crate) fn fail(
&self,
task_id: &TaskId,
error: serde_json::Value,
status_message: Option<String>,
) -> McpResult<OfficialTaskSnapshot> {
validate_json_rpc_error(&error)?;
let mut records = self.records.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "official task lifecycle lock poisoned in fail, recovering");
poisoned.into_inner()
});
let record = official_task_record_mut(&mut records, task_id)?;
let status_message = status_message.or_else(|| Some("Task execution failed".to_string()));
transition_to_terminal(
record,
OfficialTaskStatus::Failed,
status_message,
None,
Some(error),
)
}
}
#[cfg(test)]
fn generate_official_task_id() -> McpResult<TaskId> {
let identifier = draw_security_identifier().map_err(|error| {
McpError::internal_error(format!("Task identifier generation failed: {error}"))
})?;
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(identifier.as_bytes());
debug_assert_eq!(
encoded.len(),
43,
"a 256-bit task ID must be 43 base64url bytes"
);
Ok(TaskId::from_string(encoded))
}
#[cfg(test)]
fn official_task_timestamp() -> String {
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
}
#[cfg(test)]
fn official_task_record_mut<'a>(
records: &'a mut HashMap<TaskId, OfficialTaskRecord>,
task_id: &TaskId,
) -> McpResult<&'a mut OfficialTaskRecord> {
let expired = records
.get(task_id)
.is_some_and(|record| record.expires_at <= Instant::now());
if expired {
records.remove(task_id);
}
records
.get_mut(task_id)
.ok_or_else(|| McpError::invalid_params("Task not found"))
}
#[cfg(test)]
fn can_transition_official_task(from: OfficialTaskStatus, to: OfficialTaskStatus) -> bool {
matches!(
(from, to),
(
OfficialTaskStatus::Working,
OfficialTaskStatus::InputRequired
| OfficialTaskStatus::Completed
| OfficialTaskStatus::Failed
| OfficialTaskStatus::Cancelled
) | (
OfficialTaskStatus::InputRequired,
OfficialTaskStatus::Working
| OfficialTaskStatus::Failed
| OfficialTaskStatus::Cancelled
)
)
}
#[cfg(test)]
fn invalid_official_task_transition(from: OfficialTaskStatus, to: OfficialTaskStatus) -> McpError {
McpError::invalid_params(format!(
"Invalid official task transition from {from:?} to {to:?}"
))
}
#[cfg(test)]
fn touch_official_task(record: &mut OfficialTaskRecord) -> McpResult<()> {
record.update_revision = record
.update_revision
.checked_add(1)
.ok_or_else(|| McpError::internal_error("Task update revision exhausted"))?;
record.snapshot.last_updated_at = official_task_timestamp();
Ok(())
}
#[cfg(test)]
fn advance_official_task(
record: &mut OfficialTaskRecord,
status: OfficialTaskStatus,
) -> McpResult<()> {
if !can_transition_official_task(record.snapshot.status, status) {
return Err(invalid_official_task_transition(
record.snapshot.status,
status,
));
}
touch_official_task(record)?;
record.snapshot.status = status;
Ok(())
}
#[cfg(test)]
fn transition_to_terminal(
record: &mut OfficialTaskRecord,
status: OfficialTaskStatus,
status_message: Option<String>,
result: Option<serde_json::Value>,
error: Option<serde_json::Value>,
) -> McpResult<OfficialTaskSnapshot> {
debug_assert!(status.is_terminal());
advance_official_task(record, status)?;
record.snapshot.status_message = status_message;
record.snapshot.input_requests = None;
record.snapshot.result = result;
record.snapshot.error = error;
Ok(record.snapshot.clone())
}
#[cfg(test)]
fn validate_official_task_input_request(
key: &str,
request: &OfficialTaskInputRequest,
) -> McpResult<()> {
if key.is_empty() || key.len() > 256 {
return Err(McpError::invalid_params(
"Task input request keys must be non-empty and at most 256 bytes",
));
}
if !request.params.is_object() {
return Err(McpError::invalid_params(
"Task input request parameters must be an object",
));
}
Ok(())
}
#[cfg(test)]
fn validate_final_tool_result(result: &serde_json::Value) -> McpResult<()> {
let result = result
.as_object()
.ok_or_else(|| McpError::invalid_params("Completed task result must be an object"))?;
if result.get("resultType") != Some(&serde_json::Value::String("complete".to_string())) {
return Err(McpError::invalid_params(
"Completed task result must be a final complete result",
));
}
if !result
.get("content")
.is_some_and(serde_json::Value::is_array)
{
return Err(McpError::invalid_params(
"Completed task result must contain tool content",
));
}
if result
.get("isError")
.is_some_and(|is_error| !is_error.is_boolean())
{
return Err(McpError::invalid_params(
"Completed task isError must be a boolean when present",
));
}
Ok(())
}
#[cfg(test)]
fn validate_json_rpc_error(error: &serde_json::Value) -> McpResult<()> {
let error = error
.as_object()
.ok_or_else(|| McpError::invalid_params("Failed task error must be an object"))?;
let Some(code) = error.get("code") else {
return Err(McpError::invalid_params(
"Failed task error must include a JSON-RPC code",
));
};
if !code.is_i64() {
return Err(McpError::invalid_params(
"Failed task error code must be an integer",
));
}
if !error
.get("message")
.is_some_and(serde_json::Value::is_string)
{
return Err(McpError::invalid_params(
"Failed task error must include a message",
));
}
Ok(())
}
pub trait FinalTaskStore: Send + Sync {
fn execution_limits(&self) -> McpResult<FinalTaskExecutionLimits> {
Err(McpError::internal_error(
"Final task store does not implement bounded execution",
))
}
fn task_execution_budget_if_current(
&self,
_expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskExecutionBudget>> {
Err(McpError::internal_error(
"Final task store does not implement execution budget lookup",
))
}
fn expire_task_execution_if_current(
&self,
_expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskSnapshot>> {
Err(McpError::internal_error(
"Final task store does not implement execution expiry",
))
}
fn next_expired_execution_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
Err(McpError::internal_error(
"Final task store does not implement execution expiry recovery",
))
}
fn create_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()>;
fn create_task_with_work(
&self,
_task: FinalTask,
_notification: FinalTaskStatusNotification,
_work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<FinalTaskSnapshot> {
Err(McpError::internal_error(
"Final task store does not implement atomic task-work creation",
))
}
fn create_task_with_authenticated_work(
&self,
_task: FinalTask,
_notification: FinalTaskStatusNotification,
_work_descriptor: FinalTaskWorkDescriptor,
_principal: Sha256Digest,
) -> McpResult<FinalTaskSnapshot> {
Err(McpError::internal_error(
"Final task store does not implement atomic authenticated task creation",
))
}
fn get_task(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTask>>;
fn get_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTaskSnapshot>>;
fn replace_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()>;
fn replace_task_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<bool>;
fn replace_task_and_append_input_if_current(
&self,
_expected: &FinalTaskSnapshot,
_task: FinalTask,
_notification: FinalTaskStatusNotification,
_input_responses: FinalTaskInputResponses,
) -> McpResult<Option<FinalTaskSnapshot>> {
Err(McpError::internal_error(
"Final task store does not implement atomic task-input append",
))
}
fn replace_task_and_clear_input_if_current(
&self,
_expected: &FinalTaskSnapshot,
_task: FinalTask,
_notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
Err(McpError::internal_error(
"Final task store does not implement atomic task-input clearing",
))
}
fn replace_task_and_clear_input_for_handoff_if_current(
&self,
_expected: &FinalTaskSnapshot,
_owner_id: &str,
_dispatch_fence: u64,
_cancellation_required: bool,
_task: FinalTask,
_notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
Err(McpError::internal_error(
"Final task store does not implement fenced handoff transitions",
))
}
fn take_input_if_current(
&self,
_expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskInputResponses>> {
Err(McpError::internal_error(
"Final task store does not implement atomic task-input consumption",
))
}
fn take_input_for_owner_if_current(
&self,
_expected: &FinalTaskSnapshot,
_owner_id: &str,
) -> McpResult<Option<FinalTaskInputResponses>> {
Err(McpError::internal_error(
"Final task store does not implement atomic task-input consumption",
))
}
fn work_descriptor_if_current(
&self,
_expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
Err(McpError::internal_error(
"Final task store does not implement task-work lookup",
))
}
fn take_input_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskAcceptedInputClaim>> {
let Some(work_descriptor) = self.work_descriptor_if_current(expected)? else {
return Ok(None);
};
let Some(input_responses) = self.take_input_for_owner_if_current(expected, owner_id)?
else {
return Ok(None);
};
Ok(Some(FinalTaskAcceptedInputClaim::new(
expected.task().base().task_id.clone(),
expected.generation(),
owner_id,
work_descriptor,
input_responses,
)))
}
fn next_initial_work_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
Err(McpError::internal_error(
"Final task store does not implement initial-work recovery",
))
}
fn next_initial_work_snapshot_after(
&self,
_after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
Err(McpError::internal_error(
"Final task store does not implement initial-work recovery",
))
}
fn take_initial_work_if_current(
&self,
_expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
Err(McpError::internal_error(
"Final task store does not implement initial-work claiming",
))
}
fn take_initial_work_for_owner_if_current(
&self,
_expected: &FinalTaskSnapshot,
_owner_id: &str,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
Err(McpError::internal_error(
"Final task store does not implement initial-work claiming",
))
}
fn take_initial_work_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskInitialWorkClaim>> {
Ok(self
.take_initial_work_for_owner_if_current(expected, owner_id)?
.map(|work_descriptor| {
FinalTaskInitialWorkClaim::new(
expected.task().base().task_id.clone(),
expected.generation(),
owner_id,
work_descriptor,
)
}))
}
fn restore_initial_work_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
_work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Final task store does not implement initial-work restoration",
))
}
fn restore_initial_work_for_owner_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
_owner_id: &str,
_dispatch_fence: Option<u64>,
_work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Final task store does not implement initial-work restoration",
))
}
fn next_accepted_input_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
Err(McpError::internal_error(
"Final task store does not implement accepted-input recovery",
))
}
fn next_accepted_input_snapshot_after(
&self,
_after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
Err(McpError::internal_error(
"Final task store does not implement accepted-input recovery",
))
}
fn restore_input_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
_input_responses: FinalTaskInputResponses,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Final task store does not implement accepted-input restoration",
))
}
fn restore_input_for_owner_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
_owner_id: &str,
_dispatch_fence: Option<u64>,
_input_responses: FinalTaskInputResponses,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Final task store does not implement accepted-input restoration",
))
}
fn begin_handoff_dispatch_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Final task store does not implement atomic handoff dispatch election",
))
}
fn begin_handoff_dispatch_for_owner_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
_owner_id: &str,
) -> McpResult<Option<u64>> {
Err(McpError::internal_error(
"Final task store does not implement atomic handoff dispatch election",
))
}
fn renew_handoff_dispatch_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
_owner_id: &str,
_dispatch_fence: u64,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Final task store does not implement durable handoff lease renewal",
))
}
fn handoff_dispatch_lease_heartbeat_interval(&self) -> McpResult<StdDuration> {
Err(McpError::internal_error(
"Final task store does not disclose a durable handoff lease heartbeat interval",
))
}
fn finish_handoff_dispatch_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Final task store does not implement atomic handoff dispatch completion",
))
}
fn finish_handoff_dispatch_for_owner_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
_owner_id: &str,
_dispatch_fence: u64,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Final task store does not implement atomic handoff dispatch completion",
))
}
fn request_cancellation_and_clear_input_if_current(
&self,
_expected: &FinalTaskSnapshot,
_cancelled_task: FinalTask,
_cancelled_notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
Err(McpError::internal_error(
"Final task store does not implement atomic task cancellation",
))
}
fn request_cancellation(&self, task_id: &FinalTaskId) -> McpResult<()>;
fn request_cancellation_if_current(&self, expected: &FinalTaskSnapshot) -> McpResult<bool>;
fn is_cancellation_requested(&self, task_id: &FinalTaskId) -> McpResult<bool>;
fn retention_clock_now(&self) -> Instant;
fn task_retention_deadline_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>>;
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FinalTaskRetentionDeadline {
Finite(Instant),
Unlimited,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct FinalTaskExecutionLimits {
total_runtime: StdDuration,
attempt_runtime: StdDuration,
max_attempts: u32,
}
impl FinalTaskExecutionLimits {
pub fn new(
total_runtime: StdDuration,
attempt_runtime: StdDuration,
max_attempts: u32,
) -> McpResult<Self> {
if total_runtime.is_zero()
|| total_runtime > StdDuration::from_hours(720)
|| attempt_runtime.is_zero()
|| attempt_runtime > StdDuration::from_hours(24)
|| !(1..=64).contains(&max_attempts)
{
return Err(McpError::invalid_params(
"Task execution limits require positive total <= 30 days, attempt <= 24 hours, and attempts <= 64",
));
}
Ok(Self {
total_runtime,
attempt_runtime,
max_attempts,
})
}
#[must_use]
pub const fn total_runtime(self) -> StdDuration {
self.total_runtime
}
#[must_use]
pub const fn attempt_runtime(self) -> StdDuration {
self.attempt_runtime
}
#[must_use]
pub const fn max_attempts(self) -> u32 {
self.max_attempts
}
}
impl Default for FinalTaskExecutionLimits {
fn default() -> Self {
Self {
total_runtime: StdDuration::from_hours(24),
attempt_runtime: StdDuration::from_hours(1),
max_attempts: 8,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct FinalTaskExecutionBudget {
total_deadline: Instant,
attempt_deadline: Option<Instant>,
attempts: u32,
limits: FinalTaskExecutionLimits,
}
impl FinalTaskExecutionBudget {
pub fn new(
total_deadline: Instant,
attempt_deadline: Option<Instant>,
attempts: u32,
limits: FinalTaskExecutionLimits,
) -> McpResult<Self> {
if attempts > limits.max_attempts
|| attempt_deadline.is_some_and(|deadline| deadline > total_deadline)
|| (attempts == 0 && attempt_deadline.is_some())
{
return Err(McpError::internal_error(
"Invalid final task execution accounting",
));
}
Ok(Self {
total_deadline,
attempt_deadline,
attempts,
limits,
})
}
#[must_use]
pub const fn total_deadline(self) -> Instant {
self.total_deadline
}
#[must_use]
pub const fn attempt_deadline(self) -> Option<Instant> {
self.attempt_deadline
}
#[must_use]
pub const fn attempts(self) -> u32 {
self.attempts
}
#[must_use]
pub const fn limits(self) -> FinalTaskExecutionLimits {
self.limits
}
}
#[derive(Clone, Debug)]
pub struct FinalTaskSnapshot {
task: FinalTask,
generation: u64,
authenticated_principal: Option<Sha256Digest>,
}
impl FinalTaskSnapshot {
#[must_use]
pub fn new(task: FinalTask, generation: u64) -> Self {
Self {
task,
generation,
authenticated_principal: None,
}
}
#[must_use]
pub const fn with_authenticated_principal(mut self, principal: Option<Sha256Digest>) -> Self {
self.authenticated_principal = principal;
self
}
#[must_use]
pub const fn authenticated_principal(&self) -> Option<Sha256Digest> {
self.authenticated_principal
}
#[must_use]
pub const fn task(&self) -> &FinalTask {
&self.task
}
#[must_use]
pub const fn generation(&self) -> u64 {
self.generation
}
#[must_use]
pub fn into_task(self) -> FinalTask {
self.task
}
#[cfg(test)]
fn task_id(&self) -> &FinalTaskId {
&self.task.base().task_id
}
}
pub const DEFAULT_IN_MEMORY_FINAL_TASKS: usize = 1_024;
pub const MAX_IN_MEMORY_FINAL_TASK_INPUT_KEYS: usize = 1_024;
pub const MAX_IN_MEMORY_FINAL_TASK_INPUT_KEY_BYTES: usize = 64 * 1_024;
const MAX_FINAL_TASK_DESCRIPTOR_BYTES: usize = 256 * 1_024;
const MAX_FINAL_TASK_INPUT_BYTES: usize = 1_024 * 1_024;
const MAX_FINAL_TASK_RESULT_BYTES: usize = 8 * 1_024 * 1_024;
const MAX_FINAL_TASK_METADATA_BYTES: usize = 1_024 * 1_024;
const MAX_FINAL_TASK_APPLICATION_BYTES: usize = 10 * 1_024 * 1_024;
const FINAL_TASK_CANCELLATION_METADATA_RESERVE_BYTES: usize = 2 + 15;
const FINAL_TASK_CANCELLATION_RESERVE_BYTES: usize =
2 * (FINAL_TASK_CANCELLATION_METADATA_RESERVE_BYTES + 256);
pub struct InMemoryFinalTaskStore {
max_tasks: usize,
execution_limits: FinalTaskExecutionLimits,
clock: Arc<dyn Fn() -> Instant + Send + Sync>,
state: Mutex<InMemoryFinalTaskState>,
}
#[derive(Default)]
struct InMemoryFinalTaskState {
tasks: BTreeMap<FinalTaskId, FinalTask>,
authenticated_principals: BTreeMap<FinalTaskId, Sha256Digest>,
generations: BTreeMap<FinalTaskId, u64>,
next_generation: u64,
next_dispatch_fence: u64,
work_descriptors: BTreeMap<FinalTaskId, FinalTaskWorkDescriptor>,
initial_work: BTreeMap<FinalTaskId, FinalTaskWorkDescriptor>,
accepted_inputs: BTreeMap<FinalTaskId, FinalTaskInputResponses>,
input_key_history: BTreeMap<FinalTaskId, InMemoryFinalTaskInputKeyHistory>,
handoff_leases: BTreeMap<FinalTaskId, InMemoryFinalTaskHandoffLease>,
cancellation_requests: BTreeSet<FinalTaskId>,
latest_notifications: BTreeMap<FinalTaskId, FinalTaskStatusNotification>,
expires_at: BTreeMap<FinalTaskId, Instant>,
execution_budgets: BTreeMap<FinalTaskId, FinalTaskExecutionBudget>,
payload_accounting: InMemoryFinalTaskPayloadAccounting,
}
#[derive(Clone, Copy, Debug, serde::Serialize)]
struct InMemoryFinalTaskPayloadCharge {
total: usize,
initial_work: usize,
accepted_inputs: usize,
cancellation_reserve: usize,
}
struct InMemoryFinalTaskPayloadAdmission {
total: usize,
reserved: usize,
}
struct InMemoryFinalTaskPayloadAccounting {
limit: usize,
total: usize,
reserved: usize,
records: BTreeMap<FinalTaskId, InMemoryFinalTaskPayloadCharge>,
}
impl Default for InMemoryFinalTaskPayloadAccounting {
fn default() -> Self {
Self {
limit: InMemoryFinalTaskStore::DEFAULT_MAX_PAYLOAD_BYTES,
total: 0,
reserved: 0,
records: BTreeMap::new(),
}
}
}
impl InMemoryFinalTaskPayloadAccounting {
fn admit(
&self,
task_id: &FinalTaskId,
replacement: InMemoryFinalTaskPayloadCharge,
) -> McpResult<InMemoryFinalTaskPayloadAdmission> {
let previous = self.records.get(task_id);
let total = self
.total
.checked_sub(previous.map_or(0, |charge| charge.total))
.and_then(|total| total.checked_add(replacement.total));
let reserved = self
.reserved
.checked_sub(previous.map_or(0, |charge| charge.cancellation_reserve))
.and_then(|reserved| reserved.checked_add(replacement.cancellation_reserve));
match total.zip(reserved) {
Some((total, reserved))
if total
.checked_add(reserved)
.is_some_and(|bytes| bytes <= self.limit) =>
{
Ok(InMemoryFinalTaskPayloadAdmission { total, reserved })
}
_ => Err(McpError::invalid_params(
"In-memory final task payload byte capacity reached",
)),
}
}
fn commit(
&mut self,
task_id: FinalTaskId,
replacement: InMemoryFinalTaskPayloadCharge,
admission: InMemoryFinalTaskPayloadAdmission,
) {
self.records.insert(task_id, replacement);
self.total = admission.total;
self.reserved = admission.reserved;
}
fn release_handoff(&mut self, task_id: &FinalTaskId, initial: bool, accepted: bool) {
if let Some(charge) = self.records.get_mut(task_id) {
let mut released = 0;
if initial {
released += charge.initial_work;
charge.initial_work = 0;
}
if accepted {
released += charge.accepted_inputs;
charge.accepted_inputs = 0;
}
charge.total -= released;
self.total -= released;
}
}
fn release_task(&mut self, task_id: &FinalTaskId) {
if let Some(charge) = self.records.remove(task_id) {
self.total -= charge.total;
self.reserved -= charge.cancellation_reserve;
}
}
}
struct FinalTaskPayloadByteCounter {
bytes: usize,
limit: usize,
}
impl std::io::Write for FinalTaskPayloadByteCounter {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
let next = self
.bytes
.checked_add(bytes.len())
.filter(|next| *next <= self.limit)
.ok_or_else(|| std::io::Error::other("Final task payload byte limit exceeded"))?;
self.bytes = next;
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
fn final_task_payload_bytes(
value: &(impl serde::Serialize + ?Sized),
limit: usize,
member: &str,
) -> McpResult<usize> {
let mut counter = FinalTaskPayloadByteCounter { bytes: 0, limit };
serde_json::to_writer(&mut counter, value).map_err(|_| {
McpError::invalid_params(format!(
"Final task {member} exceeds its encoded payload limit"
))
})?;
Ok(counter.bytes)
}
struct FinalTaskProspectiveInputs<'a> {
current: Option<&'a FinalTaskInputResponses>,
appended: &'a FinalTaskInputResponses,
}
impl serde::Serialize for FinalTaskProspectiveInputs<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeMap as _;
let mut map = serializer.serialize_map(None)?;
if let Some(current) = self.current {
for (key, value) in current {
if !self.appended.contains_key(key) {
map.serialize_entry(key, value)?;
}
}
}
for (key, value) in self.appended {
map.serialize_entry(key, value)?;
}
map.end()
}
}
fn prepare_in_memory_final_task_payload_charge(
task: &FinalTask,
notification: &FinalTaskStatusNotification,
descriptor: Option<&FinalTaskWorkDescriptor>,
initial_work_retained: bool,
accepted_input_bytes: usize,
input_key_history: Option<&InMemoryFinalTaskInputKeyHistory>,
) -> McpResult<InMemoryFinalTaskPayloadCharge> {
let live = matches!(
task,
FinalTask::Working(_) | FinalTask::InputRequired { .. }
);
let metadata_limit = if live {
MAX_FINAL_TASK_METADATA_BYTES - FINAL_TASK_CANCELLATION_METADATA_RESERVE_BYTES
} else {
MAX_FINAL_TASK_METADATA_BYTES
};
final_task_payload_bytes(task.base(), metadata_limit, "metadata")?;
let outstanding_bytes = match task {
FinalTask::InputRequired { input_requests, .. } => {
final_task_payload_bytes(input_requests, MAX_FINAL_TASK_INPUT_BYTES, "input requests")?
}
FinalTask::Completed { result, .. } => {
final_task_payload_bytes(result, MAX_FINAL_TASK_RESULT_BYTES, "terminal result")?;
0
}
FinalTask::Failed { error, .. } => {
final_task_payload_bytes(error, MAX_FINAL_TASK_RESULT_BYTES, "terminal error")?;
0
}
FinalTask::Working(_) | FinalTask::Cancelled(_) => 0,
};
outstanding_bytes
.checked_add(accepted_input_bytes)
.filter(|bytes| *bytes <= MAX_FINAL_TASK_INPUT_BYTES)
.ok_or_else(|| {
McpError::invalid_params("Final task combined input payload limit exceeded")
})?;
let descriptor_bytes = descriptor
.map(|descriptor| {
final_task_payload_bytes(
descriptor.as_value(),
MAX_FINAL_TASK_DESCRIPTOR_BYTES,
"work descriptor",
)
})
.transpose()?
.unwrap_or(0);
let task_bytes = final_task_payload_bytes(task, MAX_FINAL_TASK_APPLICATION_BYTES, "task")?;
let application_bytes = task_bytes
.checked_add(descriptor_bytes)
.and_then(|bytes| bytes.checked_add(accepted_input_bytes))
.filter(|bytes| *bytes <= MAX_FINAL_TASK_APPLICATION_BYTES)
.ok_or_else(|| {
McpError::invalid_params("Final task aggregate application payload limit exceeded")
})?;
let notification_limit = MAX_FINAL_TASK_APPLICATION_BYTES + MAX_FINAL_TASK_METADATA_BYTES
- if live {
FINAL_TASK_CANCELLATION_METADATA_RESERVE_BYTES
} else {
0
};
let notification_bytes =
final_task_payload_bytes(notification, notification_limit, "notification")?;
let history_bytes = input_key_history
.map(|history| {
final_task_payload_bytes(
&history.keys,
MAX_FINAL_TASK_METADATA_BYTES,
"input key history",
)
})
.transpose()?
.unwrap_or(0);
let initial_work_bytes = if initial_work_retained {
descriptor_bytes
} else {
0
};
let total = application_bytes
.checked_add(notification_bytes)
.and_then(|bytes| bytes.checked_add(initial_work_bytes))
.and_then(|bytes| bytes.checked_add(history_bytes))
.ok_or_else(|| {
McpError::invalid_params("Final task retained payload byte count overflow")
})?;
Ok(InMemoryFinalTaskPayloadCharge {
total,
initial_work: initial_work_bytes,
accepted_inputs: accepted_input_bytes,
cancellation_reserve: if live {
FINAL_TASK_CANCELLATION_RESERVE_BYTES
} else {
0
},
})
}
#[derive(Clone, Default)]
struct InMemoryFinalTaskInputKeyHistory {
keys: BTreeSet<String>,
key_bytes: usize,
}
fn prepare_in_memory_final_task_input_key_history(
history: Option<&InMemoryFinalTaskInputKeyHistory>,
current: Option<&FinalTask>,
replacement: &FinalTask,
) -> McpResult<Option<InMemoryFinalTaskInputKeyHistory>> {
let FinalTask::InputRequired { input_requests, .. } = replacement else {
return Ok(None);
};
let outstanding = match current {
Some(FinalTask::InputRequired { input_requests, .. }) => Some(input_requests),
_ => None,
};
let mut next = history.cloned().unwrap_or_default();
for (key, request) in input_requests {
if let Some(existing) = outstanding.and_then(|requests| requests.get(key)) {
let existing = serde_json::to_value(existing).map_err(|error| {
McpError::internal_error(format!(
"Could not encode outstanding final task input request for comparison: {error}"
))
})?;
let replacement = serde_json::to_value(request).map_err(|error| {
McpError::internal_error(format!(
"Could not encode replacement final task input request for comparison: {error}"
))
})?;
if existing != replacement {
return Err(McpError::invalid_params(
"Task input request keys cannot be reused",
));
}
continue;
}
if next.keys.contains(key) {
return Err(McpError::invalid_params(
"Task input request keys cannot be reused",
));
}
let key_bytes = next
.key_bytes
.checked_add(key.len())
.filter(|bytes| *bytes <= MAX_IN_MEMORY_FINAL_TASK_INPUT_KEY_BYTES)
.ok_or_else(|| {
McpError::invalid_params("Task input request key history byte limit exceeded")
})?;
if next.keys.len() >= MAX_IN_MEMORY_FINAL_TASK_INPUT_KEYS {
return Err(McpError::invalid_params(
"Task input request key history count limit exceeded",
));
}
next.keys.insert(key.clone());
next.key_bytes = key_bytes;
}
Ok(Some(next))
}
struct InMemoryFinalTaskHandoffLease {
generation: u64,
kind: InMemoryFinalTaskHandoffKind,
dispatch_elected: bool,
owner_id: String,
dispatch_fence: Option<u64>,
recovery_expires_at: Option<Instant>,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum InMemoryFinalTaskHandoffKind {
Initial,
Resumed,
}
const IN_MEMORY_FINAL_TASK_HANDOFF_LEASE: StdDuration = StdDuration::from_secs(30);
const IN_MEMORY_FINAL_TASK_HANDOFF_HEARTBEAT: StdDuration = StdDuration::from_secs(10);
impl InMemoryFinalTaskStore {
pub const DEFAULT_MAX_PAYLOAD_BYTES: usize = 64 * 1_024 * 1_024;
pub const HARD_MAX_PAYLOAD_BYTES: usize = 512 * 1_024 * 1_024;
pub fn new(max_tasks: usize) -> McpResult<Self> {
Self::with_clock(max_tasks, Arc::new(Instant::now))
}
pub fn with_payload_capacity(max_tasks: usize, max_payload_bytes: usize) -> McpResult<Self> {
Self::with_clock_and_payload_capacity(max_tasks, max_payload_bytes, Arc::new(Instant::now))
}
pub fn with_clock(
max_tasks: usize,
clock: Arc<dyn Fn() -> Instant + Send + Sync>,
) -> McpResult<Self> {
Self::with_clock_and_payload_capacity(max_tasks, Self::DEFAULT_MAX_PAYLOAD_BYTES, clock)
}
pub fn with_clock_and_payload_capacity(
max_tasks: usize,
max_payload_bytes: usize,
clock: Arc<dyn Fn() -> Instant + Send + Sync>,
) -> McpResult<Self> {
Self::with_clock_payload_and_execution_limits(
max_tasks,
max_payload_bytes,
FinalTaskExecutionLimits::default(),
clock,
)
}
pub fn with_execution_limits(
max_tasks: usize,
execution_limits: FinalTaskExecutionLimits,
) -> McpResult<Self> {
Self::with_clock_payload_and_execution_limits(
max_tasks,
Self::DEFAULT_MAX_PAYLOAD_BYTES,
execution_limits,
Arc::new(Instant::now),
)
}
pub fn with_clock_payload_and_execution_limits(
max_tasks: usize,
max_payload_bytes: usize,
execution_limits: FinalTaskExecutionLimits,
clock: Arc<dyn Fn() -> Instant + Send + Sync>,
) -> McpResult<Self> {
if max_tasks == 0 {
return Err(McpError::invalid_params(
"In-memory final task store capacity must be positive",
));
}
if max_payload_bytes == 0 || max_payload_bytes > Self::HARD_MAX_PAYLOAD_BYTES {
return Err(McpError::invalid_params(
"In-memory final task payload capacity must be between 1 byte and 512 MiB",
));
}
Ok(Self {
max_tasks,
execution_limits,
clock,
state: Mutex::new(InMemoryFinalTaskState {
payload_accounting: InMemoryFinalTaskPayloadAccounting {
limit: max_payload_bytes,
..InMemoryFinalTaskPayloadAccounting::default()
},
..InMemoryFinalTaskState::default()
}),
})
}
#[must_use]
pub fn max_payload_bytes(&self) -> usize {
self.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.payload_accounting
.limit
}
#[must_use]
pub fn retained_payload_bytes(&self) -> usize {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
reclaim_expired_in_memory_final_tasks(&mut state, (self.clock)());
state.payload_accounting.total
}
#[must_use]
pub const fn max_tasks(&self) -> usize {
self.max_tasks
}
#[must_use]
pub fn task_count(&self) -> usize {
let now = (self.clock)();
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
reclaim_expired_in_memory_final_tasks(&mut state, now);
state.tasks.len()
}
#[must_use]
pub fn latest_notification(
&self,
task_id: &FinalTaskId,
) -> Option<FinalTaskStatusNotification> {
let now = (self.clock)();
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
reclaim_expired_in_memory_final_tasks(&mut state, now);
state.latest_notifications.get(task_id).cloned()
}
fn persist_task_with_work(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
work_descriptor: FinalTaskWorkDescriptor,
principal: Option<Sha256Digest>,
) -> McpResult<FinalTaskSnapshot> {
let task_id = task.base().task_id.clone();
validate_final_task_storage_shape(&task)?;
ensure_final_task_notification_matches_task(&task, ¬ification)?;
if !matches!(task, FinalTask::Working(_)) {
return Err(McpError::invalid_params(
"Initial application work requires a working final task",
));
}
validate_final_task_runtime_durations(&task)?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
let expires_at = in_memory_final_task_expiry(&task, now)?;
let execution_budget =
new_in_memory_final_task_execution_budget(self.execution_limits, now)?;
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.tasks.contains_key(&task_id) {
return Err(McpError::invalid_params("Task already exists"));
}
if state.tasks.len() == self.max_tasks {
return Err(McpError::invalid_params(
"In-memory final task store capacity reached",
));
}
let payload_charge = prepare_in_memory_final_task_payload_charge(
&task,
¬ification,
Some(&work_descriptor),
true,
0,
None,
)?;
let retained_payload_bytes = state.payload_accounting.admit(&task_id, payload_charge)?;
let generation = next_in_memory_final_task_generation(&mut state)?;
state
.payload_accounting
.commit(task_id.clone(), payload_charge, retained_payload_bytes);
state
.latest_notifications
.insert(task_id.clone(), notification);
state.tasks.insert(task_id.clone(), task);
state.generations.insert(task_id.clone(), generation);
state
.execution_budgets
.insert(task_id.clone(), execution_budget);
state
.work_descriptors
.insert(task_id.clone(), work_descriptor.clone());
state.initial_work.insert(task_id.clone(), work_descriptor);
if let Some(principal) = principal {
state
.authenticated_principals
.insert(task_id.clone(), principal);
}
if let Some(expires_at) = expires_at {
state.expires_at.insert(task_id.clone(), expires_at);
}
committed_in_memory_final_task_snapshot(&state, &task_id)
}
}
impl Default for InMemoryFinalTaskStore {
fn default() -> Self {
Self::new(DEFAULT_IN_MEMORY_FINAL_TASKS)
.expect("the fixed default in-memory final task capacity is positive")
}
}
impl FinalTaskStore for InMemoryFinalTaskStore {
fn execution_limits(&self) -> McpResult<FinalTaskExecutionLimits> {
Ok(self.execution_limits)
}
fn task_execution_budget_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskExecutionBudget>> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
reclaim_expired_in_memory_final_tasks(&mut state, (self.clock)());
let task_id = &expected.task().base().task_id;
if state.generations.get(task_id) != Some(&expected.generation()) {
return Ok(None);
}
state
.execution_budgets
.get(task_id)
.copied()
.map(Some)
.ok_or_else(|| {
McpError::internal_error("In-memory final task is missing execution accounting")
})
}
fn expire_task_execution_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskSnapshot>> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
let task_id = &expected.task().base().task_id;
if state.generations.get(task_id) != Some(&expected.generation()) {
return Ok(None);
}
let Some(reason) = in_memory_final_task_execution_exhaustion(&state, task_id, now) else {
return Ok(None);
};
let retained = state.tasks.get(task_id).ok_or_else(|| {
McpError::internal_error("In-memory final task is missing its retained task")
})?;
let task = FinalTask::Failed {
base: transition_terminal_final_task_base(
retained.base().clone(),
FinalTaskStatus::Failed,
None,
)?,
error: FinalTaskError {
code: (-32000).into(),
message: reason.to_owned(),
data: None,
additional: BTreeMap::new(),
},
};
replace_in_memory_final_task(
&mut state,
task.clone(),
final_task_notification(&task),
now,
InMemoryFinalTaskInputMutation::ExecutionExpiry,
)?;
committed_in_memory_final_task_snapshot(&state, task_id).map(Some)
}
fn next_expired_execution_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
let Some(task_id) = state.tasks.keys().find(|task_id| {
in_memory_final_task_execution_exhaustion(&state, task_id, now).is_some()
}) else {
return Ok(None);
};
committed_in_memory_final_task_snapshot(&state, task_id).map(Some)
}
fn create_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
let task_id = task.base().task_id.clone();
validate_final_task_storage_shape(&task)?;
ensure_final_task_notification_matches_task(&task, ¬ification)?;
validate_final_task_runtime_durations(&task)?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
let expires_at = in_memory_final_task_expiry(&task, now)?;
let execution_budget =
new_in_memory_final_task_execution_budget(self.execution_limits, now)?;
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.tasks.contains_key(&task_id) {
return Err(McpError::invalid_params("Task already exists"));
}
if state.tasks.len() == self.max_tasks {
return Err(McpError::invalid_params(
"In-memory final task store capacity reached",
));
}
let input_key_history = prepare_in_memory_final_task_input_key_history(None, None, &task)?;
let payload_charge = prepare_in_memory_final_task_payload_charge(
&task,
¬ification,
None,
false,
0,
input_key_history.as_ref(),
)?;
let retained_payload_bytes = state.payload_accounting.admit(&task_id, payload_charge)?;
let generation = next_in_memory_final_task_generation(&mut state)?;
state
.payload_accounting
.commit(task_id.clone(), payload_charge, retained_payload_bytes);
if let Some(history) = input_key_history {
state.input_key_history.insert(task_id.clone(), history);
}
state
.latest_notifications
.insert(task_id.clone(), notification);
state.tasks.insert(task_id.clone(), task);
state.generations.insert(task_id.clone(), generation);
state
.execution_budgets
.insert(task_id.clone(), execution_budget);
if let Some(expires_at) = expires_at {
state.expires_at.insert(task_id, expires_at);
}
Ok(())
}
fn create_task_with_work(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<FinalTaskSnapshot> {
self.persist_task_with_work(task, notification, work_descriptor, None)
}
fn create_task_with_authenticated_work(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
work_descriptor: FinalTaskWorkDescriptor,
principal: Sha256Digest,
) -> McpResult<FinalTaskSnapshot> {
self.persist_task_with_work(task, notification, work_descriptor, Some(principal))
}
fn get_task(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTask>> {
let now = (self.clock)();
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
reclaim_expired_in_memory_final_tasks(&mut state, now);
Ok(state.tasks.get(task_id).cloned())
}
fn get_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTaskSnapshot>> {
let now = (self.clock)();
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
reclaim_expired_in_memory_final_tasks(&mut state, now);
let Some(task) = state.tasks.get(task_id).cloned() else {
return Ok(None);
};
let generation = state.generations.get(task_id).copied().ok_or_else(|| {
McpError::internal_error("In-memory final task store is missing a task generation")
})?;
Ok(Some(
FinalTaskSnapshot::new(task, generation)
.with_authenticated_principal(state.authenticated_principals.get(task_id).copied()),
))
}
fn replace_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
let task_id = task.base().task_id.clone();
ensure_final_task_notification_matches_task(&task, ¬ification)?;
validate_final_task_runtime_durations(&task)?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if !state.tasks.contains_key(&task_id) {
return Err(McpError::invalid_params("Task not found"));
}
replace_in_memory_final_task(
&mut state,
task,
notification,
now,
InMemoryFinalTaskInputMutation::Clear,
)?;
Ok(())
}
fn replace_task_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<bool> {
let task_id = task.base().task_id.clone();
if expected.task().base().task_id != task_id {
return Err(McpError::invalid_params(
"Expected and replacement final task IDs must match",
));
}
ensure_final_task_notification_matches_task(&task, ¬ification)?;
validate_final_task_runtime_durations(&task)?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.generations.get(&task_id) != Some(&expected.generation()) {
return Ok(false);
}
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.generations.get(&task_id) != Some(&expected.generation()) {
return Ok(false);
}
replace_in_memory_final_task(
&mut state,
task,
notification,
now,
InMemoryFinalTaskInputMutation::Clear,
)?;
Ok(true)
}
fn replace_task_and_append_input_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
input_responses: FinalTaskInputResponses,
) -> McpResult<Option<FinalTaskSnapshot>> {
let task_id = task.base().task_id.clone();
if expected.task().base().task_id != task_id {
return Err(McpError::invalid_params(
"Expected and replacement final task IDs must match",
));
}
ensure_final_task_notification_matches_task(&task, ¬ification)?;
validate_final_task_runtime_durations(&task)?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.generations.get(&task_id) != Some(&expected.generation()) {
return Ok(None);
}
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.generations.get(&task_id) != Some(&expected.generation()) {
return Ok(None);
}
replace_in_memory_final_task(
&mut state,
task,
notification,
now,
InMemoryFinalTaskInputMutation::Append(input_responses),
)?;
committed_in_memory_final_task_snapshot(&state, &task_id).map(Some)
}
fn replace_task_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
let task_id = task.base().task_id.clone();
if expected.task().base().task_id != task_id {
return Err(McpError::invalid_params(
"Expected and replacement final task IDs must match",
));
}
ensure_final_task_notification_matches_task(&task, ¬ification)?;
validate_final_task_runtime_durations(&task)?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.generations.get(&task_id) != Some(&expected.generation()) {
return Ok(None);
}
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.generations.get(&task_id) != Some(&expected.generation()) {
return Ok(None);
}
replace_in_memory_final_task(
&mut state,
task,
notification,
now,
InMemoryFinalTaskInputMutation::Clear,
)?;
committed_in_memory_final_task_snapshot(&state, &task_id).map(Some)
}
fn replace_task_and_clear_input_for_handoff_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
dispatch_fence: u64,
cancellation_required: bool,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
if owner_id.is_empty() {
return Err(McpError::invalid_params(
"Final task handoff owner must be non-empty",
));
}
let task_id = task.base().task_id.clone();
if expected.task().base().task_id != task_id {
return Err(McpError::invalid_params(
"Expected and replacement final task IDs must match",
));
}
if matches!(&task, FinalTask::Cancelled(_)) != cancellation_required {
return Err(McpError::invalid_params(
"Fenced final task cancellation disposition does not match the replacement task",
));
}
ensure_final_task_notification_matches_task(&task, ¬ification)?;
validate_final_task_runtime_durations(&task)?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
let owns_exact_dispatch = state.handoff_leases.get(&task_id).is_some_and(|lease| {
lease.generation == expected.generation()
&& lease.dispatch_elected
&& lease.owner_id == owner_id
&& lease.dispatch_fence == Some(dispatch_fence)
&& lease
.recovery_expires_at
.is_some_and(|expires_at| expires_at > now)
});
if state.generations.get(&task_id) != Some(&expected.generation())
|| !owns_exact_dispatch
|| state.cancellation_requests.contains(&task_id) != cancellation_required
{
return Ok(None);
}
replace_in_memory_final_task(
&mut state,
task,
notification,
now,
InMemoryFinalTaskInputMutation::Clear,
)?;
committed_in_memory_final_task_snapshot(&state, &task_id).map(Some)
}
fn take_input_if_current(
&self,
_expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskInputResponses>> {
Err(McpError::internal_error(
"Raw final task input claims require an authorized service owner",
))
}
fn take_input_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskInputResponses>> {
Ok(self
.take_input_handoff_for_owner_if_current(expected, owner_id)?
.map(|claim| claim.input_responses))
}
fn take_input_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskAcceptedInputClaim>> {
if owner_id.is_empty() {
return Err(McpError::invalid_params(
"Final task handoff owner must be non-empty",
));
}
let task_id = &expected.task().base().task_id;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.generations.get(task_id) != Some(&expected.generation())
|| !state
.tasks
.get(task_id)
.is_some_and(|task| matches!(task, FinalTask::Working(_)))
|| state.cancellation_requests.contains(task_id)
|| state.handoff_leases.contains_key(task_id)
|| in_memory_final_task_execution_exhaustion(&state, task_id, now).is_some()
{
return Ok(None);
}
let Some(input_responses) = state.accepted_inputs.get(task_id).cloned() else {
return Ok(None);
};
let work_descriptor = state
.work_descriptors
.get(task_id)
.cloned()
.ok_or_else(|| {
McpError::internal_error(
"In-memory final task store is missing a working task descriptor",
)
})?;
insert_in_memory_final_task_handoff_lease(
&mut state,
task_id.clone(),
expected.generation(),
InMemoryFinalTaskHandoffKind::Resumed,
owner_id,
now,
)?;
Ok(Some(FinalTaskAcceptedInputClaim::new(
task_id.clone(),
expected.generation(),
owner_id,
work_descriptor,
input_responses,
)))
}
fn work_descriptor_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
let task_id = &expected.task().base().task_id;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.generations.get(task_id) != Some(&expected.generation())
|| !state
.tasks
.get(task_id)
.is_some_and(|task| matches!(task, FinalTask::Working(_)))
|| state.cancellation_requests.contains(task_id)
{
return Ok(None);
}
state
.work_descriptors
.get(task_id)
.cloned()
.map(Some)
.ok_or_else(|| {
McpError::internal_error(
"In-memory final task store is missing a working task descriptor",
)
})
}
fn next_initial_work_snapshot_after(
&self,
after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
let Some(task_id) = next_in_memory_final_task_recovery_id(
state.initial_work.keys(),
after_task_id,
|task_id| {
matches!(state.tasks.get(task_id), Some(FinalTask::Working(_)))
&& !state.cancellation_requests.contains(task_id)
&& !state.handoff_leases.contains_key(task_id)
&& in_memory_final_task_execution_exhaustion(&state, task_id, now).is_none()
},
) else {
return Ok(None);
};
let task = state.tasks.get(&task_id).cloned().ok_or_else(|| {
McpError::internal_error(
"In-memory final task store retained initial work for a missing task",
)
})?;
let generation = state.generations.get(&task_id).copied().ok_or_else(|| {
McpError::internal_error(
"In-memory final task store retained initial work without a task generation",
)
})?;
Ok(Some(
FinalTaskSnapshot::new(task, generation).with_authenticated_principal(
state.authenticated_principals.get(&task_id).copied(),
),
))
}
fn next_initial_work_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
self.next_initial_work_snapshot_after(None)
}
fn take_initial_work_if_current(
&self,
_expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
Err(McpError::internal_error(
"Raw final task initial-work claims require an authorized service owner",
))
}
fn take_initial_work_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
Ok(self
.take_initial_work_handoff_for_owner_if_current(expected, owner_id)?
.map(|claim| claim.work_descriptor))
}
fn take_initial_work_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskInitialWorkClaim>> {
if owner_id.is_empty() {
return Err(McpError::invalid_params(
"Final task handoff owner must be non-empty",
));
}
let task_id = &expected.task().base().task_id;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.generations.get(task_id) != Some(&expected.generation())
|| !state
.tasks
.get(task_id)
.is_some_and(|task| matches!(task, FinalTask::Working(_)))
|| state.cancellation_requests.contains(task_id)
|| state.handoff_leases.contains_key(task_id)
|| in_memory_final_task_execution_exhaustion(&state, task_id, now).is_some()
{
return Ok(None);
}
let Some(work_descriptor) = state.initial_work.get(task_id).cloned() else {
return Ok(None);
};
insert_in_memory_final_task_handoff_lease(
&mut state,
task_id.clone(),
expected.generation(),
InMemoryFinalTaskHandoffKind::Initial,
owner_id,
now,
)?;
Ok(Some(FinalTaskInitialWorkClaim::new(
task_id.clone(),
expected.generation(),
owner_id,
work_descriptor,
)))
}
fn restore_initial_work_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
_work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Raw final task initial-work restoration requires an authorized service owner",
))
}
fn restore_initial_work_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: Option<u64>,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
let owns_matching_lease = state.handoff_leases.get(task_id).is_some_and(|lease| {
lease.generation == generation
&& lease.kind == InMemoryFinalTaskHandoffKind::Initial
&& lease.owner_id == owner_id
&& lease.dispatch_fence == dispatch_fence
&& lease
.recovery_expires_at
.is_some_and(|expires_at| expires_at > now)
});
if !owns_matching_lease
|| in_memory_final_task_execution_exhaustion(&state, task_id, now).is_some()
{
return Ok(false);
}
if state.cancellation_requests.contains(task_id)
|| state.generations.get(task_id) != Some(&generation)
|| !state
.tasks
.get(task_id)
.is_some_and(|task| matches!(task, FinalTask::Working(_)))
|| state.initial_work.get(task_id) != Some(&work_descriptor)
{
return Ok(false);
}
state.handoff_leases.remove(task_id);
if let Some(budget) = state.execution_budgets.get_mut(task_id) {
budget.attempt_deadline = None;
}
Ok(true)
}
fn next_accepted_input_snapshot_after(
&self,
after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
let Some(task_id) = next_in_memory_final_task_recovery_id(
state.accepted_inputs.keys(),
after_task_id,
|task_id| {
matches!(state.tasks.get(task_id), Some(FinalTask::Working(_)))
&& !state.cancellation_requests.contains(task_id)
&& !state.handoff_leases.contains_key(task_id)
&& in_memory_final_task_execution_exhaustion(&state, task_id, now).is_none()
},
) else {
return Ok(None);
};
let task = state.tasks.get(&task_id).cloned().ok_or_else(|| {
McpError::internal_error("In-memory final task store retained input for a missing task")
})?;
let generation = state.generations.get(&task_id).copied().ok_or_else(|| {
McpError::internal_error(
"In-memory final task store retained input without a task generation",
)
})?;
Ok(Some(
FinalTaskSnapshot::new(task, generation).with_authenticated_principal(
state.authenticated_principals.get(&task_id).copied(),
),
))
}
fn next_accepted_input_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
self.next_accepted_input_snapshot_after(None)
}
fn restore_input_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
_input_responses: FinalTaskInputResponses,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Raw final task input restoration requires an authorized service owner",
))
}
fn restore_input_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: Option<u64>,
input_responses: FinalTaskInputResponses,
) -> McpResult<bool> {
if input_responses.is_empty() {
return Err(McpError::internal_error(
"Cannot restore an empty accepted-input handoff",
));
}
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
let owns_matching_lease = state.handoff_leases.get(task_id).is_some_and(|lease| {
lease.generation == generation
&& lease.kind == InMemoryFinalTaskHandoffKind::Resumed
&& lease.owner_id == owner_id
&& lease.dispatch_fence == dispatch_fence
&& lease
.recovery_expires_at
.is_some_and(|expires_at| expires_at > now)
});
if !owns_matching_lease
|| in_memory_final_task_execution_exhaustion(&state, task_id, now).is_some()
{
return Ok(false);
}
if state.cancellation_requests.contains(task_id)
|| state.generations.get(task_id) != Some(&generation)
|| !state
.tasks
.get(task_id)
.is_some_and(|task| matches!(task, FinalTask::Working(_)))
|| state.accepted_inputs.get(task_id) != Some(&input_responses)
{
return Ok(false);
}
state.handoff_leases.remove(task_id);
if let Some(budget) = state.execution_budgets.get_mut(task_id) {
budget.attempt_deadline = None;
}
Ok(true)
}
fn begin_handoff_dispatch_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Raw final task dispatch election requires an authorized service owner",
))
}
fn begin_handoff_dispatch_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
) -> McpResult<Option<u64>> {
if owner_id.is_empty() {
return Err(McpError::invalid_params(
"Final task handoff owner must be non-empty",
));
}
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.generations.get(task_id) != Some(&generation)
|| !state
.tasks
.get(task_id)
.is_some_and(|task| matches!(task, FinalTask::Working(_)))
|| state.cancellation_requests.contains(task_id)
|| in_memory_final_task_execution_exhaustion(&state, task_id, now).is_some()
|| !state.handoff_leases.get(task_id).is_some_and(|lease| {
lease.generation == generation
&& !lease.dispatch_elected
&& lease.owner_id == owner_id
&& lease
.recovery_expires_at
.is_some_and(|expires_at| expires_at > now)
})
{
return Ok(None);
}
let dispatch_fence = next_in_memory_final_task_dispatch_fence(&mut state)?;
let dispatch_expires_at = in_memory_final_task_handoff_lease_expiry(now)?;
let lease = state.handoff_leases.get_mut(task_id).ok_or_else(|| {
McpError::internal_error("In-memory final task store lost a handoff lease")
})?;
lease.dispatch_elected = true;
lease.dispatch_fence = Some(dispatch_fence);
lease.recovery_expires_at = Some(dispatch_expires_at);
Ok(Some(dispatch_fence))
}
fn renew_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if in_memory_final_task_execution_exhaustion(&state, task_id, now).is_some() {
return Ok(false);
}
let renewed_expires_at = in_memory_final_task_handoff_lease_expiry(now)?;
let Some(lease) = state.handoff_leases.get(task_id) else {
return Ok(false);
};
if lease.generation != generation
|| !lease.dispatch_elected
|| lease.owner_id != owner_id
|| lease.dispatch_fence != Some(dispatch_fence)
|| lease
.recovery_expires_at
.is_none_or(|expires_at| expires_at <= now)
|| state.generations.get(task_id) != Some(&generation)
|| !state
.tasks
.get(task_id)
.is_some_and(|task| matches!(task, FinalTask::Working(_)))
{
return Ok(false);
}
let Some(lease) = state.handoff_leases.get_mut(task_id) else {
return Ok(false);
};
lease.recovery_expires_at = Some(renewed_expires_at);
Ok(true)
}
fn handoff_dispatch_lease_heartbeat_interval(&self) -> McpResult<StdDuration> {
Ok(IN_MEMORY_FINAL_TASK_HANDOFF_HEARTBEAT)
}
fn finish_handoff_dispatch_if_current(
&self,
_task_id: &FinalTaskId,
_generation: u64,
) -> McpResult<bool> {
Err(McpError::internal_error(
"Raw final task dispatch completion requires an authorized service owner",
))
}
fn finish_handoff_dispatch_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if in_memory_final_task_execution_exhaustion(&state, task_id, now).is_some() {
return Ok(false);
}
let Some(lease) = state.handoff_leases.get(task_id) else {
return Ok(false);
};
if lease.generation != generation
|| !lease.dispatch_elected
|| lease.owner_id != owner_id
|| lease.dispatch_fence != Some(dispatch_fence)
|| lease
.recovery_expires_at
.is_none_or(|expires_at| expires_at <= now)
{
return Ok(false);
}
if state.cancellation_requests.contains(task_id) {
return Ok(false);
}
let still_dispatchable = state.generations.get(task_id) == Some(&generation)
&& state
.tasks
.get(task_id)
.is_some_and(|task| matches!(task, FinalTask::Working(_)))
&& !state.cancellation_requests.contains(task_id);
let kind = lease.kind;
state.handoff_leases.remove(task_id);
if let Some(budget) = state.execution_budgets.get_mut(task_id) {
budget.attempt_deadline = None;
}
if still_dispatchable {
match kind {
InMemoryFinalTaskHandoffKind::Initial => {
state.initial_work.remove(task_id);
state
.payload_accounting
.release_handoff(task_id, true, false);
}
InMemoryFinalTaskHandoffKind::Resumed => {
state.accepted_inputs.remove(task_id);
state
.payload_accounting
.release_handoff(task_id, false, true);
}
}
}
Ok(still_dispatchable)
}
fn request_cancellation(&self, task_id: &FinalTaskId) -> McpResult<()> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if !state.tasks.contains_key(task_id) {
return Err(McpError::invalid_params("Task not found"));
}
if let Some(reason) = in_memory_final_task_execution_exhaustion(&state, task_id, now) {
return Err(McpError::invalid_params(reason));
}
record_in_memory_final_task_cancellation(&mut state, task_id)?;
Ok(())
}
fn request_cancellation_if_current(&self, expected: &FinalTaskSnapshot) -> McpResult<bool> {
let task_id = &expected.task().base().task_id;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.generations.get(task_id) != Some(&expected.generation()) {
return Ok(false);
}
if in_memory_final_task_execution_exhaustion(&state, task_id, now).is_some() {
return Ok(false);
}
record_in_memory_final_task_cancellation(&mut state, task_id)?;
Ok(true)
}
fn request_cancellation_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
cancelled_task: FinalTask,
cancelled_notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
let task_id = &expected.task().base().task_id;
if &cancelled_task.base().task_id != task_id {
return Err(McpError::invalid_params(
"Expected and cancelled final task IDs must match",
));
}
if !matches!(&cancelled_task, FinalTask::Cancelled(_)) {
return Err(McpError::invalid_params(
"Atomic task cancellation requires a cancelled final task",
));
}
ensure_final_task_notification_matches_task(&cancelled_task, &cancelled_notification)?;
validate_final_task_transition(expected.task(), &cancelled_task)?;
validate_final_task_runtime_durations(&cancelled_task)?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let now = (self.clock)();
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.generations.get(task_id) != Some(&expected.generation()) {
return Ok(None);
}
if in_memory_final_task_execution_exhaustion(&state, task_id, now).is_some() {
return Ok(None);
}
let dispatch_elected = state.handoff_leases.get(task_id).is_some_and(|lease| {
lease.generation == expected.generation() && lease.dispatch_elected
});
if dispatch_elected {
record_in_memory_final_task_cancellation(&mut state, task_id)?;
let task = state.tasks.get(task_id).cloned().ok_or_else(|| {
McpError::internal_error(
"In-memory final task store lost an elected task during cancellation",
)
})?;
let generation = state.generations.get(task_id).copied().ok_or_else(|| {
McpError::internal_error(
"In-memory final task store lost an elected task generation during cancellation",
)
})?;
return Ok(Some(
FinalTaskSnapshot::new(task, generation).with_authenticated_principal(
state.authenticated_principals.get(task_id).copied(),
),
));
}
replace_in_memory_final_task(
&mut state,
cancelled_task.clone(),
cancelled_notification,
now,
InMemoryFinalTaskInputMutation::Clear,
)?;
let generation = state.generations.get(task_id).copied().ok_or_else(|| {
McpError::internal_error("In-memory final task store lost a cancelled task generation")
})?;
Ok(Some(
FinalTaskSnapshot::new(cancelled_task, generation)
.with_authenticated_principal(state.authenticated_principals.get(task_id).copied()),
))
}
fn is_cancellation_requested(&self, task_id: &FinalTaskId) -> McpResult<bool> {
let now = (self.clock)();
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
reclaim_expired_in_memory_final_tasks(&mut state, now);
if !state.tasks.contains_key(task_id) {
return Err(McpError::invalid_params("Task not found"));
}
Ok(state.cancellation_requests.contains(task_id))
}
fn retention_clock_now(&self) -> Instant {
(self.clock)()
}
fn task_retention_deadline_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>> {
let now = (self.clock)();
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
reclaim_expired_in_memory_final_tasks(&mut state, now);
if state.generations.get(task_id) != Some(&generation) {
return Ok(None);
}
if let Some(expires_at) = state.expires_at.get(task_id).copied() {
Ok(Some(FinalTaskRetentionDeadline::Finite(expires_at)))
} else if state.tasks.contains_key(task_id) {
Ok(Some(FinalTaskRetentionDeadline::Unlimited))
} else {
Ok(None)
}
}
}
fn ensure_final_task_notification_matches_task(
task: &FinalTask,
notification: &FinalTaskStatusNotification,
) -> McpResult<()> {
let retained_task = serde_json::to_value(task).map_err(|error| {
McpError::internal_error(format!(
"Could not encode retained final task for validation: {error}"
))
})?;
let notified_task = serde_json::to_value(¬ification.params.task).map_err(|error| {
McpError::internal_error(format!(
"Could not encode final task notification for validation: {error}"
))
})?;
if notified_task != retained_task {
return Err(McpError::invalid_params(
"Final task notification must contain exactly the retained task",
));
}
Ok(())
}
fn committed_in_memory_final_task_snapshot(
state: &InMemoryFinalTaskState,
task_id: &FinalTaskId,
) -> McpResult<FinalTaskSnapshot> {
let task = state.tasks.get(task_id).cloned().ok_or_else(|| {
McpError::internal_error("In-memory final task store lost a committed task")
})?;
let generation = state.generations.get(task_id).copied().ok_or_else(|| {
McpError::internal_error("In-memory final task store lost a committed task generation")
})?;
Ok(FinalTaskSnapshot::new(task, generation)
.with_authenticated_principal(state.authenticated_principals.get(task_id).copied()))
}
fn final_tasks_match_exactly(left: &FinalTask, right: &FinalTask) -> McpResult<bool> {
let left = serde_json::to_value(left).map_err(|error| {
McpError::internal_error(format!(
"Could not encode retained final task for exact comparison: {error}"
))
})?;
let right = serde_json::to_value(right).map_err(|error| {
McpError::internal_error(format!(
"Could not encode retained final task for exact comparison: {error}"
))
})?;
Ok(left == right)
}
fn validate_final_task_runtime_durations(task: &FinalTask) -> McpResult<()> {
for (field, duration) in [
("ttlMs", task.base().ttl_ms.as_ref()),
("pollIntervalMs", task.base().poll_interval_ms.as_ref()),
] {
if let Some(duration) = duration {
duration.try_as_millis().map_err(|error| {
McpError::invalid_params(format!(
"Task {field} cannot be represented by the local millisecond runtime: {error}"
))
})?;
}
}
Ok(())
}
fn validate_final_task_storage_shape(task: &FinalTask) -> McpResult<()> {
let expected_status = match task {
FinalTask::Working(_) => FinalTaskStatus::Working,
FinalTask::InputRequired { input_requests, .. } => {
if input_requests.is_empty() {
return Err(McpError::invalid_params(
"input_required tasks require at least one input request",
));
}
FinalTaskInputLedger::from_requests(input_requests)
.map_err(|error| McpError::invalid_params(error.to_string()))?;
FinalTaskStatus::InputRequired
}
FinalTask::Completed { .. } => FinalTaskStatus::Completed,
FinalTask::Failed { .. } => FinalTaskStatus::Failed,
FinalTask::Cancelled(_) => FinalTaskStatus::Cancelled,
};
if task.base().status != expected_status {
return Err(McpError::invalid_params(
"Final task status must match its status-specific payload",
));
}
Ok(())
}
fn validate_final_task_transition(current: &FinalTask, replacement: &FinalTask) -> McpResult<()> {
validate_final_task_storage_shape(replacement)?;
let current_base = current.base();
let replacement_base = replacement.base();
if current_base.task_id != replacement_base.task_id {
return Err(McpError::invalid_params(
"Final task replacement must preserve taskId",
));
}
if current_base.created_at != replacement_base.created_at {
return Err(McpError::invalid_params(
"Final task replacement must preserve createdAt",
));
}
if current_base.ttl_ms != replacement_base.ttl_ms {
return Err(McpError::invalid_params(
"Final task replacement must preserve ttlMs",
));
}
if current_base.poll_interval_ms != replacement_base.poll_interval_ms {
return Err(McpError::invalid_params(
"Final task replacement must preserve pollIntervalMs",
));
}
let transition_is_valid = match current_base.status {
FinalTaskStatus::Working => matches!(
replacement_base.status,
FinalTaskStatus::Working
| FinalTaskStatus::InputRequired
| FinalTaskStatus::Completed
| FinalTaskStatus::Failed
| FinalTaskStatus::Cancelled
),
FinalTaskStatus::InputRequired => matches!(
replacement_base.status,
FinalTaskStatus::Working
| FinalTaskStatus::InputRequired
| FinalTaskStatus::Completed
| FinalTaskStatus::Failed
| FinalTaskStatus::Cancelled
),
FinalTaskStatus::Completed | FinalTaskStatus::Failed | FinalTaskStatus::Cancelled => false,
};
if !transition_is_valid {
return Err(McpError::invalid_params(
"Final task replacement is not a valid lifecycle transition",
));
}
Ok(())
}
fn validate_final_task_handoff_binding(
expected: &FinalTaskSnapshot,
expected_owner_id: &str,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
handoff_kind: &'static str,
) -> McpResult<()> {
if !matches!(expected.task(), FinalTask::Working(_))
|| task_id != &expected.task().base().task_id
|| generation != expected.generation()
|| owner_id != expected_owner_id
|| owner_id.is_empty()
{
return Err(McpError::internal_error(format!(
"Final task store returned a {handoff_kind} handoff for the wrong task, generation, or owner"
)));
}
Ok(())
}
fn validate_final_task_work_descriptor(work_descriptor: &FinalTaskWorkDescriptor) -> McpResult<()> {
if work_descriptor.as_value().is_null() {
return Err(McpError::internal_error(
"Final task store returned a null application work descriptor",
));
}
Ok(())
}
fn stale_final_task_handoff_error() -> McpError {
McpError::invalid_params(
"Final task handoff is no longer the elected generation and dispatch fence",
)
}
fn in_memory_final_task_expiry(task: &FinalTask, now: Instant) -> McpResult<Option<Instant>> {
let Some(ttl_ms) = task.base().ttl_ms.as_ref() else {
return Ok(None);
};
let ttl_ms = ttl_ms.try_as_millis().map_err(|error| {
McpError::invalid_params(format!(
"Task ttlMs cannot be represented by the local millisecond runtime: {error}"
))
})?;
now.checked_add(StdDuration::from_millis(ttl_ms))
.map(Some)
.ok_or_else(|| McpError::internal_error("Task TTL exceeds process-local clock range"))
}
fn next_in_memory_final_task_generation(state: &mut InMemoryFinalTaskState) -> McpResult<u64> {
let generation = state.next_generation.checked_add(1).ok_or_else(|| {
McpError::internal_error("In-memory final task generation space is exhausted")
})?;
state.next_generation = generation;
Ok(generation)
}
fn next_in_memory_final_task_dispatch_fence(state: &mut InMemoryFinalTaskState) -> McpResult<u64> {
let fence = state.next_dispatch_fence.checked_add(1).ok_or_else(|| {
McpError::internal_error("In-memory final task dispatch fence space is exhausted")
})?;
state.next_dispatch_fence = fence;
Ok(fence)
}
fn next_in_memory_final_task_recovery_id<'a>(
task_ids: impl Iterator<Item = &'a FinalTaskId>,
after_task_id: Option<&FinalTaskId>,
mut eligible: impl FnMut(&FinalTaskId) -> bool,
) -> Option<FinalTaskId> {
let task_ids = task_ids.collect::<Vec<_>>();
after_task_id
.and_then(|after_task_id| {
task_ids
.iter()
.copied()
.find(|task_id| *task_id > after_task_id && eligible(task_id))
})
.or_else(|| task_ids.into_iter().find(|task_id| eligible(task_id)))
.cloned()
}
fn record_in_memory_final_task_cancellation(
state: &mut InMemoryFinalTaskState,
task_id: &FinalTaskId,
) -> McpResult<()> {
let generation = state.generations.get(task_id).copied().ok_or_else(|| {
McpError::internal_error("In-memory final task store is missing a task generation")
})?;
let dispatch_elected = state
.handoff_leases
.get(task_id)
.is_some_and(|lease| lease.generation == generation && lease.dispatch_elected);
let needs_generation_fence =
!dispatch_elected && !state.cancellation_requests.contains(task_id);
let next_generation = needs_generation_fence
.then(|| next_in_memory_final_task_generation(state))
.transpose()?;
state.accepted_inputs.remove(task_id);
state.initial_work.remove(task_id);
state
.payload_accounting
.release_handoff(task_id, true, true);
if !dispatch_elected {
state.handoff_leases.remove(task_id);
}
state.cancellation_requests.insert(task_id.clone());
if let Some(next_generation) = next_generation {
state.generations.insert(task_id.clone(), next_generation);
}
Ok(())
}
enum InMemoryFinalTaskInputMutation {
Clear,
ExecutionExpiry,
Append(FinalTaskInputResponses),
}
fn replace_in_memory_final_task(
state: &mut InMemoryFinalTaskState,
task: FinalTask,
notification: FinalTaskStatusNotification,
now: Instant,
input_mutation: InMemoryFinalTaskInputMutation,
) -> McpResult<()> {
let task_id = task.base().task_id.clone();
if !matches!(
input_mutation,
InMemoryFinalTaskInputMutation::ExecutionExpiry
) && let Some(reason) = in_memory_final_task_execution_exhaustion(state, &task_id, now)
{
return Err(McpError::invalid_params(reason));
}
let current = state
.tasks
.get(&task_id)
.ok_or_else(|| McpError::invalid_params("Task not found"))?;
validate_final_task_transition(current, &task)?;
let input_key_history = prepare_in_memory_final_task_input_key_history(
state.input_key_history.get(&task_id),
Some(current),
&task,
)?;
let accepted_input_bytes = match &input_mutation {
InMemoryFinalTaskInputMutation::Clear | InMemoryFinalTaskInputMutation::ExecutionExpiry => {
0
}
InMemoryFinalTaskInputMutation::Append(input_responses) => {
let current_inputs = state.accepted_inputs.get(&task_id);
if input_responses.is_empty() && current_inputs.is_none() {
0
} else {
final_task_payload_bytes(
&FinalTaskProspectiveInputs {
current: current_inputs,
appended: input_responses,
},
MAX_FINAL_TASK_INPUT_BYTES,
"accepted input",
)?
}
}
};
let working = matches!(&task, FinalTask::Working(_));
let payload_charge = prepare_in_memory_final_task_payload_charge(
&task,
¬ification,
state.work_descriptors.get(&task_id),
working && state.initial_work.contains_key(&task_id),
accepted_input_bytes,
input_key_history
.as_ref()
.or_else(|| state.input_key_history.get(&task_id)),
)?;
let retained_payload_bytes = state.payload_accounting.admit(&task_id, payload_charge)?;
let generation = next_in_memory_final_task_generation(state)?;
state
.payload_accounting
.commit(task_id.clone(), payload_charge, retained_payload_bytes);
if let Some(history) = input_key_history {
state.input_key_history.insert(task_id.clone(), history);
}
let terminal = matches!(
&task,
FinalTask::Completed { .. } | FinalTask::Failed { .. } | FinalTask::Cancelled(_)
);
state
.latest_notifications
.insert(task_id.clone(), notification);
state.tasks.insert(task_id.clone(), task);
state.generations.insert(task_id.clone(), generation);
state.handoff_leases.remove(&task_id);
if let Some(budget) = state.execution_budgets.get_mut(&task_id) {
budget.attempt_deadline = None;
}
if !working {
state.initial_work.remove(&task_id);
}
match input_mutation {
InMemoryFinalTaskInputMutation::Clear | InMemoryFinalTaskInputMutation::ExecutionExpiry => {
state.accepted_inputs.remove(&task_id);
}
InMemoryFinalTaskInputMutation::Append(input_responses) => {
if !input_responses.is_empty() {
state
.accepted_inputs
.entry(task_id.clone())
.or_default()
.extend(input_responses);
}
}
}
if terminal {
state.initial_work.remove(&task_id);
state.cancellation_requests.remove(&task_id);
}
Ok(())
}
fn reclaim_expired_in_memory_final_tasks(state: &mut InMemoryFinalTaskState, now: Instant) {
let expired_handoff_task_ids = state
.handoff_leases
.iter()
.filter(|(_, lease)| {
lease
.recovery_expires_at
.is_some_and(|expires_at| expires_at <= now)
})
.map(|(task_id, _)| task_id.clone())
.collect::<Vec<_>>();
for task_id in expired_handoff_task_ids {
if in_memory_final_task_execution_exhaustion(state, &task_id, now).is_some() {
continue;
}
let Some(lease_generation) = state
.handoff_leases
.get(&task_id)
.map(|lease| lease.generation)
else {
continue;
};
let cancellation_requires_retirement = state.cancellation_requests.contains(&task_id)
&& state
.tasks
.get(&task_id)
.is_some_and(|task| matches!(task, FinalTask::Working(_)));
if cancellation_requires_retirement {
let _ = terminalize_expired_in_memory_final_task_cancellation(state, &task_id, now);
continue;
}
let still_recoverable = state.generations.get(&task_id) == Some(&lease_generation)
&& state
.tasks
.get(&task_id)
.is_some_and(|task| matches!(task, FinalTask::Working(_)))
&& !state.cancellation_requests.contains(&task_id);
if still_recoverable {
if let Ok(generation) = next_in_memory_final_task_generation(state) {
state.handoff_leases.remove(&task_id);
if let Some(budget) = state.execution_budgets.get_mut(&task_id) {
budget.attempt_deadline = None;
}
state.generations.insert(task_id, generation);
}
} else {
state.handoff_leases.remove(&task_id);
}
}
let expired_task_ids = state
.expires_at
.iter()
.filter(|(_, expires_at)| **expires_at <= now)
.map(|(task_id, _)| task_id.clone())
.collect::<Vec<_>>();
for task_id in expired_task_ids {
state.payload_accounting.release_task(&task_id);
state.expires_at.remove(&task_id);
state.execution_budgets.remove(&task_id);
state.tasks.remove(&task_id);
state.authenticated_principals.remove(&task_id);
state.generations.remove(&task_id);
state.work_descriptors.remove(&task_id);
state.initial_work.remove(&task_id);
state.accepted_inputs.remove(&task_id);
state.input_key_history.remove(&task_id);
state.handoff_leases.remove(&task_id);
state.cancellation_requests.remove(&task_id);
state.latest_notifications.remove(&task_id);
}
}
fn terminalize_expired_in_memory_final_task_cancellation(
state: &mut InMemoryFinalTaskState,
task_id: &FinalTaskId,
now: Instant,
) -> McpResult<()> {
let Some(FinalTask::Working(base)) = state.tasks.get(task_id).cloned() else {
return Err(McpError::internal_error(
"Expired cancellation lease no longer owns a working final task",
));
};
let task = FinalTask::Cancelled(transition_terminal_final_task_base(
base,
FinalTaskStatus::Cancelled,
None,
)?);
replace_in_memory_final_task(
state,
task.clone(),
final_task_notification(&task),
now,
InMemoryFinalTaskInputMutation::Clear,
)
}
fn insert_in_memory_final_task_handoff_lease(
state: &mut InMemoryFinalTaskState,
task_id: FinalTaskId,
generation: u64,
kind: InMemoryFinalTaskHandoffKind,
owner_id: &str,
now: Instant,
) -> McpResult<()> {
if let Some(reason) = in_memory_final_task_execution_exhaustion(state, &task_id, now) {
return Err(McpError::invalid_params(reason));
}
let expires_at = in_memory_final_task_handoff_lease_expiry(now)?;
let budget = state.execution_budgets.get(&task_id).ok_or_else(|| {
McpError::internal_error("In-memory final task is missing execution accounting")
})?;
let attempts = budget
.attempts
.checked_add(1)
.filter(|attempts| *attempts <= budget.limits.max_attempts)
.ok_or_else(|| McpError::invalid_params(FINAL_TASK_ATTEMPTS_EXHAUSTED))?;
let attempt_deadline = now
.checked_add(budget.limits.attempt_runtime)
.ok_or_else(|| McpError::internal_error("Task attempt exceeds process-local clock range"))?
.min(budget.total_deadline);
if state.handoff_leases.contains_key(&task_id) {
return Err(McpError::internal_error(
"In-memory final task store overwrote a live handoff lease",
));
}
let budget = state.execution_budgets.get_mut(&task_id).ok_or_else(|| {
McpError::internal_error("In-memory final task is missing execution accounting")
})?;
budget.attempts = attempts;
budget.attempt_deadline = Some(attempt_deadline);
if state
.handoff_leases
.insert(
task_id,
InMemoryFinalTaskHandoffLease {
generation,
kind,
dispatch_elected: false,
owner_id: owner_id.to_owned(),
dispatch_fence: None,
recovery_expires_at: Some(expires_at),
},
)
.is_some()
{
return Err(McpError::internal_error(
"In-memory final task store overwrote a live handoff lease",
));
}
Ok(())
}
const FINAL_TASK_TOTAL_DEADLINE_EXCEEDED: &str = "Task total execution deadline exceeded";
const FINAL_TASK_ATTEMPT_DEADLINE_EXCEEDED: &str = "Task attempt execution deadline exceeded";
const FINAL_TASK_ATTEMPTS_EXHAUSTED: &str = "Task execution attempt limit exceeded";
fn new_in_memory_final_task_execution_budget(
limits: FinalTaskExecutionLimits,
now: Instant,
) -> McpResult<FinalTaskExecutionBudget> {
let total_deadline = now.checked_add(limits.total_runtime).ok_or_else(|| {
McpError::internal_error("Task total execution exceeds process-local clock range")
})?;
FinalTaskExecutionBudget::new(total_deadline, None, 0, limits)
}
fn in_memory_final_task_execution_exhaustion(
state: &InMemoryFinalTaskState,
task_id: &FinalTaskId,
now: Instant,
) -> Option<&'static str> {
let task = state.tasks.get(task_id)?;
if !matches!(
task,
FinalTask::Working(_) | FinalTask::InputRequired { .. }
) {
return None;
}
let budget = state.execution_budgets.get(task_id)?;
if now >= budget.total_deadline {
return Some(FINAL_TASK_TOTAL_DEADLINE_EXCEEDED);
}
if budget
.attempt_deadline
.is_some_and(|deadline| now >= deadline)
{
return Some(FINAL_TASK_ATTEMPT_DEADLINE_EXCEEDED);
}
if budget.attempts == budget.limits.max_attempts
&& !state.handoff_leases.contains_key(task_id)
&& matches!(task, FinalTask::Working(_))
&& (state.initial_work.contains_key(task_id) || state.accepted_inputs.contains_key(task_id))
{
return Some(FINAL_TASK_ATTEMPTS_EXHAUSTED);
}
None
}
fn in_memory_final_task_handoff_lease_expiry(now: Instant) -> McpResult<Instant> {
now.checked_add(IN_MEMORY_FINAL_TASK_HANDOFF_LEASE)
.ok_or_else(|| {
McpError::internal_error("Task handoff lease exceeds process-local clock range")
})
}
pub type FinalTaskNotificationEmitter = Arc<dyn Fn(FinalTaskStatusNotification) + Send + Sync>;
pub(crate) type OwnedTaskNotificationEmitter =
Arc<dyn Fn(FinalTaskStatusNotification, Option<Sha256Digest>) + Send + Sync>;
#[derive(Clone, Debug, PartialEq)]
pub struct FinalTaskWorkDescriptor(serde_json::Value);
impl FinalTaskWorkDescriptor {
pub fn new(descriptor: serde_json::Value) -> McpResult<Self> {
if descriptor.is_null() {
return Err(McpError::invalid_params(
"Final task work descriptor must identify an application operation",
));
}
Ok(Self(descriptor))
}
#[must_use]
pub const fn as_value(&self) -> &serde_json::Value {
&self.0
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct FinalTaskInitialWorkClaim {
task_id: FinalTaskId,
generation: u64,
owner_id: String,
work_descriptor: FinalTaskWorkDescriptor,
}
impl FinalTaskInitialWorkClaim {
#[must_use]
pub fn new(
task_id: FinalTaskId,
generation: u64,
owner_id: impl Into<String>,
work_descriptor: FinalTaskWorkDescriptor,
) -> Self {
Self {
task_id,
generation,
owner_id: owner_id.into(),
work_descriptor,
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct FinalTaskAcceptedInputClaim {
task_id: FinalTaskId,
generation: u64,
owner_id: String,
work_descriptor: FinalTaskWorkDescriptor,
input_responses: FinalTaskInputResponses,
}
impl FinalTaskAcceptedInputClaim {
#[must_use]
pub fn new(
task_id: FinalTaskId,
generation: u64,
owner_id: impl Into<String>,
work_descriptor: FinalTaskWorkDescriptor,
input_responses: FinalTaskInputResponses,
) -> Self {
Self {
task_id,
generation,
owner_id: owner_id.into(),
work_descriptor,
input_responses,
}
}
}
struct FinalTaskHandoffAuthority {
runtime: FinalTaskRuntime,
task_id: FinalTaskId,
generation: u64,
owner_id: String,
dispatch_fence: u64,
transition_committed: Arc<AtomicBool>,
execution_budget: FinalTaskExecutionBudget,
}
impl std::fmt::Debug for FinalTaskHandoffAuthority {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("FinalTaskHandoffAuthority")
.field("task_id", &self.task_id)
.field("generation", &self.generation)
.field("owner_id", &self.owner_id)
.field("dispatch_fence", &self.dispatch_fence)
.finish_non_exhaustive()
}
}
impl FinalTaskHandoffAuthority {
fn require_input(
&self,
input_requests: FinalTaskInputRequests,
status_message: Option<String>,
) -> McpResult<FinalTask> {
let task = self.runtime.fenced_require_input(
&self.task_id,
self.generation,
&self.owner_id,
self.dispatch_fence,
input_requests,
status_message,
)?;
self.transition_committed
.store(true, TaskServiceOrdering::Release);
Ok(task)
}
fn complete_task(
&self,
result: FinalTaskCallToolResult,
status_message: Option<String>,
) -> McpResult<FinalTask> {
let task = self.runtime.fenced_complete_task(
&self.task_id,
self.generation,
&self.owner_id,
self.dispatch_fence,
result,
status_message,
)?;
self.transition_committed
.store(true, TaskServiceOrdering::Release);
Ok(task)
}
fn fail_task(
&self,
error: FinalTaskError,
status_message: Option<String>,
) -> McpResult<FinalTask> {
let task = self.runtime.fenced_fail_task(
&self.task_id,
self.generation,
&self.owner_id,
self.dispatch_fence,
error,
status_message,
)?;
self.transition_committed
.store(true, TaskServiceOrdering::Release);
Ok(task)
}
fn honor_cancellation(&self, status_message: Option<String>) -> McpResult<FinalTask> {
let task = self.runtime.fenced_honor_cancellation(
&self.task_id,
self.generation,
&self.owner_id,
self.dispatch_fence,
status_message,
)?;
self.transition_committed
.store(true, TaskServiceOrdering::Release);
Ok(task)
}
fn is_cancellation_requested(&self) -> McpResult<bool> {
self.runtime
.fenced_cancellation_requested(&self.task_id, self.generation)
}
}
#[derive(Debug)]
#[must_use = "initial task work must be handed to the application supervisor"]
pub struct FinalTaskInitialWork {
task_id: FinalTaskId,
generation: u64,
work_descriptor: FinalTaskWorkDescriptor,
authority: Option<FinalTaskHandoffAuthority>,
}
impl PartialEq for FinalTaskInitialWork {
fn eq(&self, other: &Self) -> bool {
self.task_id == other.task_id
&& self.generation == other.generation
&& self.work_descriptor == other.work_descriptor
}
}
impl FinalTaskInitialWork {
#[must_use]
pub const fn task_id(&self) -> &FinalTaskId {
&self.task_id
}
#[must_use]
pub const fn generation(&self) -> u64 {
self.generation
}
#[must_use]
pub const fn work_descriptor(&self) -> &FinalTaskWorkDescriptor {
&self.work_descriptor
}
pub fn require_input(
&self,
input_requests: FinalTaskInputRequests,
status_message: Option<String>,
) -> McpResult<FinalTask> {
self.authority()?
.require_input(input_requests, status_message)
}
pub fn complete_task(
&self,
result: FinalTaskCallToolResult,
status_message: Option<String>,
) -> McpResult<FinalTask> {
self.authority()?.complete_task(result, status_message)
}
pub fn fail_task(
&self,
error: FinalTaskError,
status_message: Option<String>,
) -> McpResult<FinalTask> {
self.authority()?.fail_task(error, status_message)
}
pub fn is_cancellation_requested(&self) -> McpResult<bool> {
self.authority()?.is_cancellation_requested()
}
pub fn honor_cancellation(&self, status_message: Option<String>) -> McpResult<FinalTask> {
self.authority()?.honor_cancellation(status_message)
}
fn authority(&self) -> McpResult<&FinalTaskHandoffAuthority> {
self.authority.as_ref().ok_or_else(|| {
McpError::internal_error(
"Final task application mutations require an elected service handoff",
)
})
}
fn attach_authority(&mut self, authority: FinalTaskHandoffAuthority) {
self.authority = Some(authority);
}
fn restore_copy(&self) -> FinalTaskWorkDescriptor {
self.work_descriptor.clone()
}
}
#[derive(Debug)]
#[must_use = "accepted task input must be handed to the resumed worker"]
pub struct FinalTaskAcceptedInput {
task_id: FinalTaskId,
generation: u64,
work_descriptor: FinalTaskWorkDescriptor,
input_responses: FinalTaskInputResponses,
authority: Option<FinalTaskHandoffAuthority>,
}
impl PartialEq for FinalTaskAcceptedInput {
fn eq(&self, other: &Self) -> bool {
self.task_id == other.task_id
&& self.generation == other.generation
&& self.work_descriptor == other.work_descriptor
&& self.input_responses == other.input_responses
}
}
impl FinalTaskAcceptedInput {
#[must_use]
pub const fn task_id(&self) -> &FinalTaskId {
&self.task_id
}
#[must_use]
pub const fn generation(&self) -> u64 {
self.generation
}
#[must_use]
pub const fn work_descriptor(&self) -> &FinalTaskWorkDescriptor {
&self.work_descriptor
}
#[must_use]
pub const fn input_responses(&self) -> &FinalTaskInputResponses {
&self.input_responses
}
#[must_use]
pub fn into_parts(
self,
) -> (
FinalTaskId,
u64,
FinalTaskWorkDescriptor,
FinalTaskInputResponses,
) {
(
self.task_id,
self.generation,
self.work_descriptor,
self.input_responses,
)
}
pub fn require_input(
&self,
input_requests: FinalTaskInputRequests,
status_message: Option<String>,
) -> McpResult<FinalTask> {
self.authority()?
.require_input(input_requests, status_message)
}
pub fn complete_task(
&self,
result: FinalTaskCallToolResult,
status_message: Option<String>,
) -> McpResult<FinalTask> {
self.authority()?.complete_task(result, status_message)
}
pub fn fail_task(
&self,
error: FinalTaskError,
status_message: Option<String>,
) -> McpResult<FinalTask> {
self.authority()?.fail_task(error, status_message)
}
pub fn is_cancellation_requested(&self) -> McpResult<bool> {
self.authority()?.is_cancellation_requested()
}
pub fn honor_cancellation(&self, status_message: Option<String>) -> McpResult<FinalTask> {
self.authority()?.honor_cancellation(status_message)
}
fn authority(&self) -> McpResult<&FinalTaskHandoffAuthority> {
self.authority.as_ref().ok_or_else(|| {
McpError::internal_error(
"Final task application mutations require an elected service handoff",
)
})
}
fn attach_authority(&mut self, authority: FinalTaskHandoffAuthority) {
self.authority = Some(authority);
}
fn restore_copy(&self) -> FinalTaskInputResponses {
self.input_responses.clone()
}
}
#[must_use = "task supervisor handoffs must be consumed by the application"]
pub enum FinalTaskSupervisorHandoff {
Initial(FinalTaskInitialWork),
Resumed(FinalTaskAcceptedInput),
}
impl FinalTaskSupervisorHandoff {
pub fn execution_budget(&self) -> McpResult<FinalTaskExecutionBudget> {
let authority = match self {
Self::Initial(initial) => initial.authority()?,
Self::Resumed(accepted) => accepted.authority()?,
};
Ok(authority.execution_budget)
}
fn attach_authority(&mut self, authority: FinalTaskHandoffAuthority) {
match self {
Self::Initial(initial) => initial.attach_authority(authority),
Self::Resumed(accepted) => accepted.attach_authority(authority),
}
}
}
pub trait FinalTaskRetentionAuthority: Send + Sync {
fn authorize_unlimited_retention(&self) -> McpResult<()>;
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct FinalTaskRuntimeConfig {
ttl_ms: Option<u64>,
poll_interval_ms: Option<u64>,
}
impl FinalTaskRuntimeConfig {
pub fn new(ttl_ms: u64, poll_interval_ms: Option<u64>) -> McpResult<Self> {
Self::with_ttl(Some(ttl_ms), poll_interval_ms)
}
pub fn with_ttl(ttl_ms: Option<u64>, poll_interval_ms: Option<u64>) -> McpResult<Self> {
let ttl_ms = ttl_ms.ok_or_else(|| {
McpError::invalid_params("Tasks ttlMs null requires an explicit retention authority")
})?;
final_task_duration(ttl_ms)?;
if let Some(interval) = poll_interval_ms {
final_task_duration(interval)?;
}
Ok(Self {
ttl_ms: Some(ttl_ms),
poll_interval_ms,
})
}
pub fn with_unlimited_ttl(
retention_authority: &dyn FinalTaskRetentionAuthority,
poll_interval_ms: Option<u64>,
) -> McpResult<Self> {
retention_authority.authorize_unlimited_retention()?;
if let Some(interval) = poll_interval_ms {
final_task_duration(interval)?;
}
Ok(Self {
ttl_ms: None,
poll_interval_ms,
})
}
#[must_use]
pub const fn ttl_ms(&self) -> Option<u64> {
self.ttl_ms
}
}
pub type FinalTaskSupervisorFuture<'a> = Pin<Box<dyn Future<Output = McpResult<()>> + Send + 'a>>;
pub trait ApplicationTaskSupervisor: Send + Sync {
fn resume<'a>(
&'a self,
cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a>;
}
const MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN: usize = 64;
const MAX_FINAL_TASK_RECOVERY_CAS_RETRIES: usize = 64;
const FINAL_TASK_RECOVERY_WAKE_INTERVAL: StdDuration = StdDuration::from_secs(1);
pub(crate) const DEFAULT_CONCURRENT_TASK_HANDOFFS: usize = 32;
pub(crate) const MAX_CONCURRENT_TASK_HANDOFFS: usize = 256;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
enum FinalTaskRecoveryKind {
Initial,
Resumed,
}
impl FinalTaskRecoveryKind {
const fn other(self) -> Self {
match self {
Self::Initial => Self::Resumed,
Self::Resumed => Self::Initial,
}
}
}
#[cfg(test)]
const FINAL_TASK_TEST_DIRECT_OWNER: &str = "final-task-test-direct-owner";
struct FinalTaskServiceSignal {
service_id: u64,
sender: Sender<FinalTaskId>,
ready_generation: Option<u64>,
cancellation_wake: Arc<FinalTaskCancellationWake>,
}
#[derive(Default)]
struct FinalTaskCancellationWake {
state: Mutex<FinalTaskCancellationWakeState>,
}
#[derive(Default)]
struct FinalTaskCancellationWakeState {
active: BTreeMap<FinalTaskId, Option<std::task::Waker>>,
}
struct FinalTaskCancellationWakeRegistration {
wake: Arc<FinalTaskCancellationWake>,
task_id: FinalTaskId,
}
impl FinalTaskCancellationWake {
fn activate(
self: &Arc<Self>,
task_id: &FinalTaskId,
) -> McpResult<FinalTaskCancellationWakeRegistration> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.active.contains_key(task_id) {
return Err(McpError::internal_error(
"Task service attempted to execute duplicate supervisor handoffs",
));
}
if state.active.len() >= MAX_CONCURRENT_TASK_HANDOFFS {
return Err(McpError::internal_error(
"Task service cancellation registration capacity is exhausted",
));
}
state.active.insert(task_id.clone(), None);
Ok(FinalTaskCancellationWakeRegistration {
wake: Arc::clone(self),
task_id: task_id.clone(),
})
}
fn register_waker(&self, task_id: &FinalTaskId, waker: &std::task::Waker) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(registered) = state.active.get_mut(task_id) {
*registered = Some(waker.clone());
}
}
fn wake_if_active(&self, task_id: &FinalTaskId) {
let waker = {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.active.get(task_id).cloned().flatten()
};
if let Some(waker) = waker {
waker.wake();
}
}
}
impl FinalTaskCancellationWakeRegistration {
fn register_waker(&self, waker: &std::task::Waker) {
self.wake.register_waker(&self.task_id, waker);
}
}
impl Drop for FinalTaskCancellationWakeRegistration {
fn drop(&mut self) {
let mut state = self
.wake
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.active.remove(&self.task_id);
}
}
struct FinalTaskServiceReadinessLease {
runtime: FinalTaskRuntime,
service_id: u64,
ready_generation: u64,
}
pub struct AuthorizedTaskServiceRunner {
runtime: FinalTaskRuntime,
service_id: u64,
dispatch_owner: String,
receiver: Receiver<FinalTaskId>,
supervisor: Arc<dyn ApplicationTaskSupervisor>,
next_recovery_kind: FinalTaskRecoveryKind,
initial_recovery_cursor: Option<FinalTaskId>,
accepted_recovery_cursor: Option<FinalTaskId>,
max_concurrent_handoffs: usize,
}
#[derive(Clone)]
struct FinalTaskHandoffExecutor {
runtime: FinalTaskRuntime,
service_id: u64,
dispatch_owner: String,
supervisor: Arc<dyn ApplicationTaskSupervisor>,
}
struct FinalTaskConcurrentHandoff<'a> {
kind: FinalTaskRecoveryKind,
task_id: FinalTaskId,
future: FinalTaskSupervisorFuture<'a>,
wake: Arc<FinalTaskHandoffWake>,
}
struct FinalTaskHandoffWake {
ready: AtomicBool,
parent: Mutex<Option<std::task::Waker>>,
}
impl std::task::Wake for FinalTaskHandoffWake {
fn wake(self: Arc<Self>) {
self.wake_by_ref();
}
fn wake_by_ref(self: &Arc<Self>) {
self.ready.store(true, TaskServiceOrdering::Release);
let parent = self
.parent
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
if let Some(parent) = parent {
parent.wake();
}
}
}
#[derive(Clone)]
pub struct FinalTaskRuntime {
store: Arc<dyn FinalTaskStore>,
config: FinalTaskRuntimeConfig,
notification_emitters: Arc<Mutex<Vec<OwnedTaskNotificationEmitter>>>,
service_signal: Arc<Mutex<Option<FinalTaskServiceSignal>>>,
next_task_service_id: Arc<AtomicU64>,
}
impl FinalTaskRuntime {
#[must_use]
pub fn new(
store: Arc<dyn FinalTaskStore>,
config: FinalTaskRuntimeConfig,
notification_emitter: FinalTaskNotificationEmitter,
) -> Self {
Self {
store,
config,
notification_emitters: Arc::new(Mutex::new(vec![Arc::new(
move |notification, _principal| notification_emitter(notification),
)])),
service_signal: Arc::new(Mutex::new(None)),
next_task_service_id: Arc::new(AtomicU64::new(0)),
}
}
#[must_use]
pub fn in_memory(
config: FinalTaskRuntimeConfig,
notification_emitter: FinalTaskNotificationEmitter,
) -> Self {
Self::new(
Arc::new(InMemoryFinalTaskStore::default()),
config,
notification_emitter,
)
}
pub fn in_memory_with_capacity(
max_tasks: usize,
config: FinalTaskRuntimeConfig,
notification_emitter: FinalTaskNotificationEmitter,
) -> McpResult<Self> {
Ok(Self::new(
Arc::new(InMemoryFinalTaskStore::new(max_tasks)?),
config,
notification_emitter,
))
}
pub(crate) fn add_owned_notification_emitter(&self, emitter: OwnedTaskNotificationEmitter) {
self.notification_emitters
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(emitter);
}
#[cfg(test)]
fn add_notification_emitter(&self, emitter: FinalTaskNotificationEmitter) {
self.add_owned_notification_emitter(Arc::new(move |notification, _principal| {
emitter(notification);
}));
}
pub fn install_task_service(
&self,
queue_capacity: usize,
supervisor: Arc<dyn ApplicationTaskSupervisor>,
) -> McpResult<AuthorizedTaskServiceRunner> {
if queue_capacity == 0 {
return Err(McpError::invalid_params(
"Task service queue capacity must be positive",
));
}
let service_id = self
.next_task_service_id
.try_update(
TaskServiceOrdering::Relaxed,
TaskServiceOrdering::Relaxed,
|current| current.checked_add(1),
)
.map_err(|_| McpError::internal_error("Task service generation space is exhausted"))?
.checked_add(1)
.ok_or_else(|| {
McpError::internal_error("Task service generation space is exhausted")
})?;
let (sender, receiver) = mpsc::channel(queue_capacity);
#[cfg(test)]
let dispatch_owner = FINAL_TASK_TEST_DIRECT_OWNER.to_owned();
#[cfg(not(test))]
let dispatch_owner = generate_final_task_dispatch_owner()?;
let mut signal = self
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if signal.is_some() {
return Err(McpError::invalid_params(
"A task service is already installed for this runtime",
));
}
*signal = Some(FinalTaskServiceSignal {
service_id,
sender,
ready_generation: None,
cancellation_wake: Arc::new(FinalTaskCancellationWake::default()),
});
Ok(AuthorizedTaskServiceRunner {
runtime: self.clone(),
service_id,
dispatch_owner,
receiver,
supervisor,
next_recovery_kind: FinalTaskRecoveryKind::Initial,
initial_recovery_cursor: None,
accepted_recovery_cursor: None,
max_concurrent_handoffs: 1,
})
}
#[cfg(test)]
fn take_accepted_input(
&self,
task_id: &FinalTaskId,
) -> McpResult<Option<FinalTaskAcceptedInput>> {
let current = self.load_task_snapshot(task_id)?;
let claim = self
.store
.take_input_handoff_for_owner_if_current(¤t, FINAL_TASK_TEST_DIRECT_OWNER)?;
claim
.map(|claim| {
self.validate_accepted_input_claim(¤t, FINAL_TASK_TEST_DIRECT_OWNER, claim)
})
.transpose()
}
#[cfg(test)]
fn recover_accepted_input(&self) -> McpResult<Option<FinalTaskAcceptedInput>> {
for _ in 0..MAX_FINAL_TASK_RECOVERY_CAS_RETRIES {
let Some(candidate) = self.store.next_accepted_input_snapshot_after(None)? else {
return Ok(None);
};
let candidate = self.validate_loaded_task_snapshot(candidate, None)?;
let task_id = candidate.task().base().task_id.clone();
if !matches!(candidate.task(), FinalTask::Working(_)) {
return Err(McpError::internal_error(
"Final task store returned a non-working accepted-input recovery candidate",
));
}
let Some(claim) = self.store.take_input_handoff_for_owner_if_current(
&candidate,
FINAL_TASK_TEST_DIRECT_OWNER,
)?
else {
continue;
};
let handoff = self.validate_accepted_input_claim(
&candidate,
FINAL_TASK_TEST_DIRECT_OWNER,
claim,
)?;
debug_assert_eq!(handoff.task_id(), &task_id);
return Ok(Some(handoff));
}
Err(McpError::internal_error(
"Accepted-input recovery exceeded bounded lost-CAS retries",
))
}
fn recover_accepted_input_excluding(
&self,
cx: &Cx,
owner_id: &str,
recovery_cursor: &mut Option<FinalTaskId>,
excluded_task_ids: &BTreeSet<FinalTaskId>,
) -> McpResult<Option<FinalTaskAcceptedInput>> {
let mut skipped = BTreeSet::new();
let mut lost_comparisons = 0;
let mut work_steps = 0;
for _ in 0..MAX_FINAL_TASK_RECOVERY_CAS_RETRIES + excluded_task_ids.len() {
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
let Some(candidate) = self
.store
.next_accepted_input_snapshot_after(recovery_cursor.as_ref())?
else {
return Ok(None);
};
let candidate = self.validate_loaded_task_snapshot(candidate, None)?;
let task_id = candidate.task().base().task_id.clone();
if excluded_task_ids.contains(&task_id) {
if !skipped.insert(task_id.clone()) {
return Ok(None);
}
*recovery_cursor = Some(task_id);
continue;
}
work_steps += 1;
if self.expire_execution_snapshot(&candidate)?.is_some() {
*recovery_cursor = Some(task_id);
if work_steps == MAX_FINAL_TASK_RECOVERY_CAS_RETRIES {
break;
}
continue;
}
if !matches!(candidate.task(), FinalTask::Working(_)) {
return Err(McpError::internal_error(
"Final task store returned a non-working accepted-input recovery candidate",
));
}
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
let Some(claim) = self
.store
.take_input_handoff_for_owner_if_current(&candidate, owner_id)?
else {
lost_comparisons += 1;
if work_steps == MAX_FINAL_TASK_RECOVERY_CAS_RETRIES {
break;
}
continue;
};
return self
.validate_accepted_input_claim(&candidate, owner_id, claim)
.map(Some);
}
if lost_comparisons < MAX_FINAL_TASK_RECOVERY_CAS_RETRIES {
if let Some(task_id) = recovery_cursor.as_ref() {
self.signal_task_service(task_id.clone());
}
return Ok(None);
}
Err(McpError::internal_error(
"Accepted-input recovery exceeded bounded lost-CAS retries",
))
}
#[cfg(test)]
fn recover_initial_work(&self) -> McpResult<Option<FinalTaskInitialWork>> {
for _ in 0..MAX_FINAL_TASK_RECOVERY_CAS_RETRIES {
let Some(candidate) = self.store.next_initial_work_snapshot_after(None)? else {
return Ok(None);
};
let candidate = self.validate_loaded_task_snapshot(candidate, None)?;
if !matches!(candidate.task(), FinalTask::Working(_)) {
return Err(McpError::internal_error(
"Final task store returned a non-working initial-work recovery candidate",
));
}
let task_id = candidate.task().base().task_id.clone();
let Some(claim) = self.store.take_initial_work_handoff_for_owner_if_current(
&candidate,
FINAL_TASK_TEST_DIRECT_OWNER,
)?
else {
continue;
};
let handoff =
self.validate_initial_work_claim(&candidate, FINAL_TASK_TEST_DIRECT_OWNER, claim)?;
debug_assert_eq!(handoff.task_id(), &task_id);
return Ok(Some(handoff));
}
Err(McpError::internal_error(
"Initial-work recovery exceeded bounded lost-CAS retries",
))
}
fn recover_initial_work_excluding(
&self,
cx: &Cx,
owner_id: &str,
recovery_cursor: &mut Option<FinalTaskId>,
excluded_task_ids: &BTreeSet<FinalTaskId>,
) -> McpResult<Option<FinalTaskInitialWork>> {
let mut skipped = BTreeSet::new();
let mut lost_comparisons = 0;
let mut work_steps = 0;
for _ in 0..MAX_FINAL_TASK_RECOVERY_CAS_RETRIES + excluded_task_ids.len() {
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
let Some(candidate) = self
.store
.next_initial_work_snapshot_after(recovery_cursor.as_ref())?
else {
return Ok(None);
};
let candidate = self.validate_loaded_task_snapshot(candidate, None)?;
let task_id = candidate.task().base().task_id.clone();
if excluded_task_ids.contains(&task_id) {
if !skipped.insert(task_id.clone()) {
return Ok(None);
}
*recovery_cursor = Some(task_id);
continue;
}
work_steps += 1;
if self.expire_execution_snapshot(&candidate)?.is_some() {
*recovery_cursor = Some(task_id);
if work_steps == MAX_FINAL_TASK_RECOVERY_CAS_RETRIES {
break;
}
continue;
}
if !matches!(candidate.task(), FinalTask::Working(_)) {
return Err(McpError::internal_error(
"Final task store returned a non-working initial-work recovery candidate",
));
}
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
let Some(claim) = self
.store
.take_initial_work_handoff_for_owner_if_current(&candidate, owner_id)?
else {
lost_comparisons += 1;
if work_steps == MAX_FINAL_TASK_RECOVERY_CAS_RETRIES {
break;
}
continue;
};
return self
.validate_initial_work_claim(&candidate, owner_id, claim)
.map(Some);
}
if lost_comparisons < MAX_FINAL_TASK_RECOVERY_CAS_RETRIES {
if let Some(task_id) = recovery_cursor.as_ref() {
self.signal_task_service(task_id.clone());
}
return Ok(None);
}
Err(McpError::internal_error(
"Initial-work recovery exceeded bounded lost-CAS retries",
))
}
#[cfg(test)]
fn recover_initial_work_with_checkpoints(
&self,
cx: &Cx,
owner_id: &str,
after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskInitialWork>> {
self.recover_initial_work_excluding(
cx,
owner_id,
&mut after_task_id.cloned(),
&BTreeSet::new(),
)
}
fn take_initial_work_with_checkpoint(
&self,
cx: &Cx,
task_id: &FinalTaskId,
owner_id: &str,
) -> McpResult<Option<FinalTaskInitialWork>> {
let Some(current) = self.load_optional_task_snapshot(task_id)? else {
return Ok(None);
};
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
let claim = self
.store
.take_initial_work_handoff_for_owner_if_current(¤t, owner_id)?;
claim
.map(|claim| self.validate_initial_work_claim(¤t, owner_id, claim))
.transpose()
}
fn take_accepted_input_with_checkpoint(
&self,
cx: &Cx,
task_id: &FinalTaskId,
owner_id: &str,
) -> McpResult<Option<FinalTaskAcceptedInput>> {
let Some(current) = self.load_optional_task_snapshot(task_id)? else {
return Ok(None);
};
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
let claim = self
.store
.take_input_handoff_for_owner_if_current(¤t, owner_id)?;
claim
.map(|claim| self.validate_accepted_input_claim(¤t, owner_id, claim))
.transpose()
}
pub fn create_task(&self, _status_message: Option<String>) -> McpResult<CreateTaskResult> {
Err(McpError::invalid_params(
"Final task creation requires an opaque application work descriptor",
))
}
pub fn create_task_with_work(
&self,
work_descriptor: FinalTaskWorkDescriptor,
status_message: Option<String>,
) -> McpResult<CreateTaskResult> {
self.create_task_with_principal(work_descriptor, status_message, None)
}
pub(crate) fn create_task_for_request(
&self,
ctx: &McpContext,
work_descriptor: FinalTaskWorkDescriptor,
status_message: Option<String>,
) -> McpResult<CreateTaskResult> {
ctx.ensure_live()?;
self.create_task_with_principal(
work_descriptor,
status_message,
task_request_principal(ctx)?,
)
}
fn create_task_with_principal(
&self,
work_descriptor: FinalTaskWorkDescriptor,
status_message: Option<String>,
principal: Option<Sha256Digest>,
) -> McpResult<CreateTaskResult> {
let task_id = generate_final_task_id()?;
let now = final_task_timestamp()?;
let task = FinalTask::Working(FinalTaskBase {
task_id,
status: FinalTaskStatus::Working,
status_message,
created_at: now.clone(),
last_updated_at: now,
ttl_ms: self.config.ttl_ms.map(final_task_duration).transpose()?,
poll_interval_ms: self
.config
.poll_interval_ms
.map(final_task_duration)
.transpose()?,
});
self.persist_new_with_work_while_service_ready(task.clone(), work_descriptor, principal)?;
Ok(CreateTaskResult {
task,
meta: None,
additional: BTreeMap::new(),
})
}
pub fn get_task(&self, task_id: &FinalTaskId) -> McpResult<FinalGetTaskResult> {
Ok(fastmcp_protocol::CompleteTaskResult {
task: self.load_task_snapshot(task_id)?.into_task(),
meta: None,
additional: BTreeMap::new(),
})
}
fn fenced_require_input(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
input_requests: FinalTaskInputRequests,
status_message: Option<String>,
) -> McpResult<FinalTask> {
if input_requests.is_empty() {
return Err(McpError::invalid_params(
"input_required tasks require at least one input request",
));
}
FinalTaskInputLedger::from_requests(&input_requests)
.map_err(|error| McpError::invalid_params(error.to_string()))?;
let current = self.load_task_snapshot(task_id)?;
if current.generation() != generation {
return Err(stale_final_task_handoff_error());
}
let FinalTask::Working(base) = current.task() else {
return Err(McpError::invalid_params(
"only a working task can require client input",
));
};
let task = FinalTask::InputRequired {
base: transition_final_task_base(
base.clone(),
FinalTaskStatus::InputRequired,
status_message,
)?,
input_requests,
};
self.persist_fenced_handoff_transition_clearing_input(
¤t,
owner_id,
dispatch_fence,
false,
task.clone(),
)?;
Ok(task)
}
fn fenced_complete_task(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
result: FinalTaskCallToolResult,
status_message: Option<String>,
) -> McpResult<FinalTask> {
let current = self.load_task_snapshot(task_id)?;
if current.generation() != generation {
return Err(stale_final_task_handoff_error());
}
let FinalTask::Working(base) = current.task() else {
return Err(McpError::invalid_params("only a working task can complete"));
};
let task = FinalTask::Completed {
base: transition_terminal_final_task_base(
base.clone(),
FinalTaskStatus::Completed,
status_message,
)?,
result,
};
self.persist_fenced_handoff_transition_clearing_input(
¤t,
owner_id,
dispatch_fence,
false,
task.clone(),
)?;
Ok(task)
}
fn fenced_fail_task(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
error: FinalTaskError,
status_message: Option<String>,
) -> McpResult<FinalTask> {
let current = self.load_task_snapshot(task_id)?;
if current.generation() != generation {
return Err(stale_final_task_handoff_error());
}
if matches!(
current.task(),
FinalTask::Completed { .. } | FinalTask::Failed { .. } | FinalTask::Cancelled(_)
) {
return Err(McpError::invalid_params("terminal tasks cannot fail"));
}
let task = FinalTask::Failed {
base: transition_terminal_final_task_base(
current.task().base().clone(),
FinalTaskStatus::Failed,
status_message,
)?,
error,
};
self.persist_fenced_handoff_transition_clearing_input(
¤t,
owner_id,
dispatch_fence,
false,
task.clone(),
)?;
Ok(task)
}
fn fenced_honor_cancellation(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
status_message: Option<String>,
) -> McpResult<FinalTask> {
let current = self.load_task_snapshot(task_id)?;
if current.generation() != generation {
return Err(stale_final_task_handoff_error());
}
if matches!(
current.task(),
FinalTask::Completed { .. } | FinalTask::Failed { .. } | FinalTask::Cancelled(_)
) {
return Err(McpError::invalid_params(
"terminal tasks cannot be cancelled",
));
}
let task = FinalTask::Cancelled(transition_terminal_final_task_base(
current.task().base().clone(),
FinalTaskStatus::Cancelled,
status_message,
)?);
self.persist_fenced_handoff_transition_clearing_input(
¤t,
owner_id,
dispatch_fence,
true,
task.clone(),
)?;
Ok(task)
}
fn fenced_cancellation_requested(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<bool> {
let current = self.load_task_snapshot(task_id)?;
if current.generation() != generation {
return Err(stale_final_task_handoff_error());
}
self.store.is_cancellation_requested(task_id)
}
#[cfg(test)]
pub fn require_input(
&self,
task_id: &FinalTaskId,
input_requests: FinalTaskInputRequests,
status_message: Option<String>,
) -> McpResult<FinalTask> {
if input_requests.is_empty() {
return Err(McpError::invalid_params(
"input_required tasks require at least one input request",
));
}
FinalTaskInputLedger::from_requests(&input_requests)
.map_err(|error| McpError::invalid_params(error.to_string()))?;
let current = self.load_task_snapshot(task_id)?;
let FinalTask::Working(base) = current.task() else {
return Err(McpError::invalid_params(
"only a working task can require client input",
));
};
let task = FinalTask::InputRequired {
base: transition_final_task_base(
base.clone(),
FinalTaskStatus::InputRequired,
status_message,
)?,
input_requests,
};
self.persist_transition_clearing_input(¤t, task.clone())?;
Ok(task)
}
pub fn update_task(
&self,
task_id: &FinalTaskId,
input_responses: &FinalTaskInputResponses,
) -> McpResult<UpdateTaskResult> {
let current = self.load_task_snapshot(task_id)?;
self.update_task_snapshot(¤t, input_responses)
}
fn update_task_snapshot(
&self,
current: &FinalTaskSnapshot,
input_responses: &FinalTaskInputResponses,
) -> McpResult<UpdateTaskResult> {
let FinalTask::InputRequired {
base,
input_requests,
} = current.task()
else {
return Ok(UpdateTaskResult::default());
};
let mut input_requests = input_requests.clone();
let ledger = FinalTaskInputLedger::from_requests(&input_requests)
.map_err(|error| McpError::invalid_params(error.to_string()))?;
let outstanding_responses = input_responses
.iter()
.filter(|(key, _)| input_requests.contains_key(*key))
.map(|(key, response)| (key.clone(), response.clone()))
.collect::<FinalTaskInputResponses>();
if outstanding_responses.is_empty() {
return Ok(UpdateTaskResult::default());
}
ledger
.validate_responses(&outstanding_responses)
.map_err(|error| McpError::invalid_params(error.to_string()))?;
for key in outstanding_responses.keys() {
input_requests.remove(key);
}
let task = if input_requests.is_empty() {
FinalTask::Working(transition_final_task_base(
base.clone(),
FinalTaskStatus::Working,
None,
)?)
} else {
FinalTask::InputRequired {
base: transition_final_task_base(
base.clone(),
FinalTaskStatus::InputRequired,
None,
)?,
input_requests,
}
};
self.persist_transition_appending_input(current, task, outstanding_responses)?;
Ok(UpdateTaskResult::default())
}
pub fn cancel_task(&self, task_id: &FinalTaskId) -> McpResult<FinalCancelTaskResult> {
let current = self.load_task_snapshot(task_id)?;
self.cancel_task_snapshot(¤t)
}
fn cancel_task_snapshot(
&self,
current: &FinalTaskSnapshot,
) -> McpResult<FinalCancelTaskResult> {
let task_id = ¤t.task().base().task_id;
if matches!(
current.task(),
FinalTask::Completed { .. } | FinalTask::Failed { .. } | FinalTask::Cancelled(_)
) {
return Err(McpError::invalid_params(
"terminal tasks cannot be cancelled",
));
}
let cancelled_task = FinalTask::Cancelled(transition_terminal_final_task_base(
current.task().base().clone(),
FinalTaskStatus::Cancelled,
None,
)?);
let cancelled_notification = final_task_notification(&cancelled_task);
self.validate_task_transition_write(current, &cancelled_task, &cancelled_notification)?;
let cancellation = self.store.request_cancellation_and_clear_input_if_current(
current,
cancelled_task.clone(),
cancelled_notification.clone(),
)?;
let Some(cancellation) = cancellation else {
return Err(McpError::invalid_params(
"Task state changed before cancellation could be recorded",
));
};
let cancellation = self.validate_loaded_task_snapshot(cancellation, Some(task_id))?;
let terminal_cancellation =
self.validate_cancellation_store_result(current, &cancelled_task, cancellation)?;
if terminal_cancellation {
self.emit(cancelled_notification, current.authenticated_principal());
}
self.signal_task_service_cancellation(task_id.clone());
Ok(FinalCancelTaskResult::default())
}
fn validate_cancellation_store_result(
&self,
expected: &FinalTaskSnapshot,
cancelled_task: &FinalTask,
returned: FinalTaskSnapshot,
) -> McpResult<bool> {
if returned.authenticated_principal() != expected.authenticated_principal() {
return Err(McpError::internal_error(
"Final task store changed the task principal",
));
}
let terminal = match returned.task() {
FinalTask::Cancelled(_) => {
validate_final_task_transition(expected.task(), returned.task()).map_err(|_| {
McpError::internal_error(
"Final task store returned an invalid terminal cancellation transition",
)
})?;
if returned.generation() <= expected.generation()
|| !final_tasks_match_exactly(returned.task(), cancelled_task)?
{
return Err(McpError::internal_error(
"Final task store substituted the intended terminal cancellation",
));
}
true
}
FinalTask::Working(_) => {
if returned.generation() != expected.generation()
|| !final_tasks_match_exactly(returned.task(), expected.task())?
{
return Err(McpError::internal_error(
"Final task store returned a substituted active cancellation snapshot",
));
}
false
}
_ => {
return Err(McpError::internal_error(
"Final task store returned a non-working active cancellation snapshot",
));
}
};
let task_id = &expected.task().base().task_id;
let cancellation_requested = !terminal && self.store.is_cancellation_requested(task_id)?;
let committed = self.load_task_snapshot(task_id)?;
if committed.authenticated_principal() != returned.authenticated_principal() {
return Err(McpError::internal_error(
"Final task store changed the task principal",
));
}
if committed.generation() == returned.generation()
&& final_tasks_match_exactly(committed.task(), returned.task())?
{
if !terminal && !cancellation_requested {
return Err(McpError::internal_error(
"Final task store did not retain the active cancellation intent",
));
}
return Ok(terminal);
}
if terminal
|| committed.generation() <= returned.generation()
|| !matches!(committed.task(), FinalTask::Cancelled(_))
{
return Err(McpError::internal_error(
"Final task store returned cancellation data that is not durably retained",
));
}
validate_final_task_transition(returned.task(), committed.task()).map_err(|_| {
McpError::internal_error(
"Final task store returned an invalid cooperative cancellation transition",
)
})?;
Ok(false)
}
pub fn is_cancellation_requested(&self, task_id: &FinalTaskId) -> McpResult<bool> {
self.store.is_cancellation_requested(task_id)
}
#[cfg(test)]
pub fn honor_cancellation(
&self,
task_id: &FinalTaskId,
status_message: Option<String>,
) -> McpResult<FinalTask> {
if !self.store.is_cancellation_requested(task_id)? {
return Err(McpError::invalid_params(
"task cancellation has not been requested",
));
}
let current = self.load_task_snapshot(task_id)?;
if matches!(
current.task(),
FinalTask::Completed { .. } | FinalTask::Failed { .. } | FinalTask::Cancelled(_)
) {
return Err(McpError::invalid_params(
"terminal tasks cannot be cancelled",
));
}
let task = FinalTask::Cancelled(transition_terminal_final_task_base(
current.task().base().clone(),
FinalTaskStatus::Cancelled,
status_message,
)?);
self.persist_transition_clearing_input(¤t, task.clone())?;
Ok(task)
}
#[cfg(test)]
pub fn complete_task(
&self,
task_id: &FinalTaskId,
result: FinalTaskCallToolResult,
status_message: Option<String>,
) -> McpResult<FinalTask> {
let current = self.load_task_snapshot(task_id)?;
let FinalTask::Working(base) = current.task() else {
return Err(McpError::invalid_params("only a working task can complete"));
};
let task = FinalTask::Completed {
base: transition_terminal_final_task_base(
base.clone(),
FinalTaskStatus::Completed,
status_message,
)?,
result,
};
self.persist_transition_clearing_input(¤t, task.clone())?;
Ok(task)
}
#[cfg(test)]
pub fn fail_task(
&self,
task_id: &FinalTaskId,
error: FinalTaskError,
status_message: Option<String>,
) -> McpResult<FinalTask> {
let current = self.load_task_snapshot(task_id)?;
if matches!(
current.task(),
FinalTask::Completed { .. } | FinalTask::Failed { .. } | FinalTask::Cancelled(_)
) {
return Err(McpError::invalid_params("terminal tasks cannot fail"));
}
let task = FinalTask::Failed {
base: transition_terminal_final_task_base(
current.task().base().clone(),
FinalTaskStatus::Failed,
status_message,
)?,
error,
};
self.persist_transition_clearing_input(¤t, task.clone())?;
Ok(task)
}
fn load_optional_task_snapshot(
&self,
task_id: &FinalTaskId,
) -> McpResult<Option<FinalTaskSnapshot>> {
let Some(snapshot) = self.store.get_task_snapshot(task_id)? else {
return Ok(None);
};
let snapshot = self.validate_loaded_task_snapshot(snapshot, Some(task_id))?;
Ok(Some(
self.expire_execution_snapshot(&snapshot)?
.unwrap_or(snapshot),
))
}
fn expire_execution_snapshot(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskSnapshot>> {
if !matches!(
expected.task(),
FinalTask::Working(_) | FinalTask::InputRequired { .. }
) {
return Ok(None);
}
let Some(committed) = self.store.expire_task_execution_if_current(expected)? else {
return Ok(None);
};
let committed =
self.validate_loaded_task_snapshot(committed, Some(&expected.task().base().task_id))?;
if !matches!(committed.task(), FinalTask::Failed { .. })
|| committed.generation() <= expected.generation()
|| committed.authenticated_principal() != expected.authenticated_principal()
{
return Err(McpError::internal_error(
"Final task store returned an invalid execution-expiry commit",
));
}
validate_final_task_transition(expected.task(), committed.task())?;
self.emit_committed(final_task_notification(committed.task()), &committed);
Ok(Some(committed))
}
fn expire_pending_execution(&self, cx: &Cx) -> McpResult<()> {
for turn in 0..MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN {
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
let Some(candidate) = self.store.next_expired_execution_snapshot()? else {
break;
};
let candidate = self.validate_loaded_task_snapshot(candidate, None)?;
self.expire_execution_snapshot(&candidate)?;
if turn + 1 == MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN {
self.signal_task_service(candidate.task().base().task_id.clone());
}
}
Ok(())
}
fn load_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<FinalTaskSnapshot> {
self.load_optional_task_snapshot(task_id)?
.ok_or_else(|| McpError::invalid_params("Task not found"))
}
pub(crate) fn task_for_request(
&self,
ctx: &McpContext,
task_id: &FinalTaskId,
) -> McpResult<FinalTaskSnapshot> {
ctx.ensure_live()?;
let principal = task_request_principal(ctx)?;
let snapshot = self
.store
.get_task_snapshot(task_id)?
.ok_or_else(|| McpError::invalid_params("Task not found"))?;
let snapshot = self.validate_loaded_task_snapshot(snapshot, Some(task_id))?;
if snapshot.authenticated_principal() != principal {
return Err(McpError::invalid_params("Task not found"));
}
Ok(self
.expire_execution_snapshot(&snapshot)?
.unwrap_or(snapshot))
}
pub(crate) fn notification_matches_task_owner(
&self,
admitted: &FinalTaskSnapshot,
notification: &FinalTaskStatusNotification,
) -> bool {
let task_id = ¬ification.params.task.base().task_id;
if task_id != &admitted.task().base().task_id
|| notification.params.task.base().created_at != admitted.task().base().created_at
{
return false;
}
self.load_task_snapshot(task_id).is_ok_and(|current| {
current.authenticated_principal() == admitted.authenticated_principal()
&& current.task().base().created_at == admitted.task().base().created_at
})
}
fn validate_loaded_task_snapshot(
&self,
snapshot: FinalTaskSnapshot,
expected_task_id: Option<&FinalTaskId>,
) -> McpResult<FinalTaskSnapshot> {
validate_final_task_storage_shape(snapshot.task()).map_err(|error| {
McpError::internal_error(format!(
"Final task store returned an invalid durable task shape: {}",
error.message
))
})?;
validate_final_task_runtime_durations(snapshot.task()).map_err(|error| {
McpError::internal_error(format!(
"Final task store returned a task with an invalid runtime duration: {}",
error.message
))
})?;
if let Some(expected_task_id) = expected_task_id
&& &snapshot.task().base().task_id != expected_task_id
{
return Err(McpError::internal_error(
"Final task store returned a task under the wrong identifier",
));
}
Ok(snapshot)
}
fn validate_initial_work_claim(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
claim: FinalTaskInitialWorkClaim,
) -> McpResult<FinalTaskInitialWork> {
validate_final_task_handoff_binding(
expected,
owner_id,
&claim.task_id,
claim.generation,
&claim.owner_id,
"initial-work",
)?;
validate_final_task_work_descriptor(&claim.work_descriptor)?;
self.validate_claim_snapshot_still_current(expected)?;
Ok(FinalTaskInitialWork {
task_id: claim.task_id,
generation: claim.generation,
work_descriptor: claim.work_descriptor,
authority: None,
})
}
fn validate_accepted_input_claim(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
claim: FinalTaskAcceptedInputClaim,
) -> McpResult<FinalTaskAcceptedInput> {
validate_final_task_handoff_binding(
expected,
owner_id,
&claim.task_id,
claim.generation,
&claim.owner_id,
"accepted-input",
)?;
validate_final_task_work_descriptor(&claim.work_descriptor)?;
if claim.input_responses.is_empty() {
return Err(McpError::internal_error(
"Final task store returned an empty accepted-input handoff",
));
}
self.validate_claim_snapshot_still_current(expected)?;
Ok(FinalTaskAcceptedInput {
task_id: claim.task_id,
generation: claim.generation,
work_descriptor: claim.work_descriptor,
input_responses: claim.input_responses,
authority: None,
})
}
fn validate_claim_snapshot_still_current(&self, expected: &FinalTaskSnapshot) -> McpResult<()> {
let task_id = &expected.task().base().task_id;
let current = self
.store
.get_task_snapshot(task_id)?
.ok_or_else(|| McpError::invalid_params("Task not found"))?;
let current = self.validate_loaded_task_snapshot(current, Some(task_id))?;
if current.generation() != expected.generation()
|| current.authenticated_principal() != expected.authenticated_principal()
|| !final_tasks_match_exactly(current.task(), expected.task())?
{
return Err(McpError::internal_error(
"Final task store changed the claimed task before application handoff",
));
}
Ok(())
}
fn validate_committed_transition(
&self,
expected: &FinalTaskSnapshot,
intended: &FinalTask,
committed: FinalTaskSnapshot,
) -> McpResult<FinalTaskSnapshot> {
let committed =
self.validate_loaded_task_snapshot(committed, Some(&intended.base().task_id))?;
validate_final_task_transition(expected.task(), committed.task()).map_err(|_| {
McpError::internal_error(
"Final task store committed an invalid durable task transition",
)
})?;
if committed.generation() <= expected.generation()
|| committed.authenticated_principal() != expected.authenticated_principal()
|| !final_tasks_match_exactly(committed.task(), intended)?
{
return Err(McpError::internal_error(
"Final task store substituted a durable task transition",
));
}
Ok(committed)
}
fn validate_committed_new_task(
&self,
intended: &FinalTask,
principal: Option<Sha256Digest>,
committed: FinalTaskSnapshot,
) -> McpResult<FinalTaskSnapshot> {
let committed =
self.validate_loaded_task_snapshot(committed, Some(&intended.base().task_id))?;
if !final_tasks_match_exactly(committed.task(), intended)? {
return Err(McpError::internal_error(
"Final task store substituted a newly created durable task",
));
}
if committed.authenticated_principal() != principal {
return Err(McpError::internal_error(
"Final task store substituted the task principal at creation",
));
}
Ok(committed)
}
fn validate_new_task_write(
&self,
task: &FinalTask,
notification: &FinalTaskStatusNotification,
) -> McpResult<()> {
validate_final_task_storage_shape(task)?;
ensure_final_task_notification_matches_task(task, notification)?;
validate_final_task_runtime_durations(task)
}
fn validate_task_transition_write(
&self,
expected: &FinalTaskSnapshot,
task: &FinalTask,
notification: &FinalTaskStatusNotification,
) -> McpResult<()> {
self.validate_new_task_write(task, notification)?;
validate_final_task_transition(expected.task(), task)
}
#[cfg(test)]
fn persist_new_with_work(
&self,
task: FinalTask,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<()> {
let task_id = task.base().task_id.clone();
let notification = final_task_notification(&task);
self.validate_new_task_write(&task, ¬ification)?;
validate_final_task_work_descriptor(&work_descriptor)?;
let committed = self.store.create_task_with_work(
task.clone(),
notification.clone(),
work_descriptor,
)?;
let committed = self.validate_committed_new_task(&task, None, committed)?;
self.emit_committed(notification, &committed);
self.signal_task_service(task_id);
Ok(())
}
fn persist_new_with_work_while_service_ready(
&self,
task: FinalTask,
work_descriptor: FinalTaskWorkDescriptor,
principal: Option<Sha256Digest>,
) -> McpResult<()> {
let task_id = task.base().task_id.clone();
let notification = final_task_notification(&task);
self.validate_new_task_write(&task, ¬ification)?;
validate_final_task_work_descriptor(&work_descriptor)?;
self.store.execution_limits()?;
let committed = {
let signal = self
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !Self::task_service_is_ready(signal.as_ref()) {
return Err(McpError::invalid_params(
"Final task creation requires an installed ready task service",
));
}
if let Some(principal) = principal {
self.store.create_task_with_authenticated_work(
task.clone(),
notification.clone(),
work_descriptor,
principal,
)?
} else {
self.store.create_task_with_work(
task.clone(),
notification.clone(),
work_descriptor,
)?
}
};
let committed = self.validate_committed_new_task(&task, principal, committed)?;
self.emit_committed(notification, &committed);
self.signal_task_service(task_id);
Ok(())
}
fn persist_transition_appending_input(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
input_responses: FinalTaskInputResponses,
) -> McpResult<()> {
let wakeup_task_id = match &task {
FinalTask::Working(base) => Some(base.task_id.clone()),
_ => None,
};
let notification = final_task_notification(&task);
self.validate_task_transition_write(expected, &task, ¬ification)?;
let result = self.store.replace_task_and_append_input_if_current(
expected,
task.clone(),
notification.clone(),
input_responses,
);
if !matches!(&result, Ok(Some(_))) {
self.expire_execution_snapshot(expected)?;
}
let Some(committed) = result? else {
return Err(McpError::invalid_params(
"Task state changed before the transition could be recorded",
));
};
let committed = self.validate_committed_transition(expected, &task, committed)?;
self.emit_committed(notification, &committed);
if let Some(task_id) = wakeup_task_id {
self.signal_task_service(task_id);
}
Ok(())
}
#[cfg(test)]
fn persist_transition_clearing_input(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
) -> McpResult<()> {
let notification = final_task_notification(&task);
self.validate_task_transition_write(expected, &task, ¬ification)?;
let Some(committed) = self.store.replace_task_and_clear_input_if_current(
expected,
task.clone(),
notification.clone(),
)?
else {
return Err(McpError::invalid_params(
"Task state changed before the transition could be recorded",
));
};
let committed = self.validate_committed_transition(expected, &task, committed)?;
self.emit_committed(notification, &committed);
Ok(())
}
fn persist_fenced_handoff_transition_clearing_input(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
dispatch_fence: u64,
cancellation_required: bool,
task: FinalTask,
) -> McpResult<()> {
let notification = final_task_notification(&task);
self.validate_task_transition_write(expected, &task, ¬ification)?;
let result = self
.store
.replace_task_and_clear_input_for_handoff_if_current(
expected,
owner_id,
dispatch_fence,
cancellation_required,
task.clone(),
notification.clone(),
);
if !matches!(&result, Ok(Some(_))) {
self.expire_execution_snapshot(expected)?;
}
let Some(committed) = result? else {
return Err(stale_final_task_handoff_error());
};
let committed = self.validate_committed_transition(expected, &task, committed)?;
self.emit_committed(notification, &committed);
Ok(())
}
fn emit_committed(
&self,
notification: FinalTaskStatusNotification,
committed: &FinalTaskSnapshot,
) {
let current = self.load_task_snapshot(&committed.task().base().task_id);
if current.is_ok_and(|current| {
current.generation() == committed.generation()
&& current.authenticated_principal() == committed.authenticated_principal()
&& final_tasks_match_exactly(current.task(), committed.task()).unwrap_or(false)
}) {
self.emit(notification, committed.authenticated_principal());
}
}
fn emit(&self, notification: FinalTaskStatusNotification, principal: Option<Sha256Digest>) {
let emitters = self
.notification_emitters
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let mut panicked_emitter_count = 0usize;
for emitter in emitters {
if catch_unwind(AssertUnwindSafe(|| {
emitter(notification.clone(), principal);
}))
.is_err()
{
panicked_emitter_count += 1;
}
}
if panicked_emitter_count != 0 {
log::error!(
target: "fastmcp_rust::server",
"Final Task notification delivery degraded after durable mutation; panicked_emitter_count={}",
panicked_emitter_count
);
}
}
fn restore_accepted_input(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: Option<u64>,
input_responses: FinalTaskInputResponses,
) -> McpResult<bool> {
self.store.restore_input_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
input_responses,
)
}
fn restore_initial_work(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: Option<u64>,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
self.store.restore_initial_work_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
work_descriptor,
)
}
fn begin_handoff_dispatch(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
) -> McpResult<Option<u64>> {
self.store
.begin_handoff_dispatch_for_owner_if_current(task_id, generation, owner_id)
}
fn renew_handoff_dispatch(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
self.store
.renew_handoff_dispatch_if_current(task_id, generation, owner_id, dispatch_fence)
}
fn handoff_dispatch_lease_heartbeat_interval(&self) -> McpResult<StdDuration> {
let interval = self.store.handoff_dispatch_lease_heartbeat_interval()?;
if interval.is_zero() {
return Err(McpError::internal_error(
"Final task store returned a zero dispatch lease heartbeat interval",
));
}
Ok(interval)
}
fn finish_handoff_dispatch(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
self.store.finish_handoff_dispatch_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
)
}
fn retention_clock_now(&self) -> Instant {
self.store.retention_clock_now()
}
fn task_retention_deadline(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>> {
self.store
.task_retention_deadline_if_current(task_id, generation)
}
#[must_use]
pub fn is_task_service_ready(&self) -> bool {
let signal = self
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Self::task_service_is_ready(signal.as_ref())
}
pub(crate) fn has_installed_task_service(&self) -> bool {
self.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_some()
}
pub(crate) fn ensure_task_service_ready(&self) -> McpResult<()> {
if !self.is_task_service_ready() {
return Err(McpError::invalid_params(
"Final task creation requires an installed ready task service",
));
}
Ok(())
}
fn task_service_is_ready(service: Option<&FinalTaskServiceSignal>) -> bool {
service.is_some_and(|service| {
service.ready_generation == Some(service.service_id) && !service.sender.is_closed()
})
}
fn mark_task_service_ready(
&self,
service_id: u64,
) -> McpResult<FinalTaskServiceReadinessLease> {
let mut signal = self
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(service) = signal.as_mut() else {
return Err(McpError::internal_error(
"Task service runner has no installed wakeup authority",
));
};
if service.service_id != service_id {
return Err(McpError::internal_error(
"A stale task service runner cannot establish readiness",
));
}
if service.sender.is_closed() {
*signal = None;
return Err(McpError::internal_error(
"Task service wakeup authority closed before runner start",
));
}
if service.ready_generation.is_some() {
return Err(McpError::internal_error(
"Task service readiness is already owned by a live runner",
));
}
service.ready_generation = Some(service_id);
Ok(FinalTaskServiceReadinessLease {
runtime: self.clone(),
service_id,
ready_generation: service_id,
})
}
fn signal_task_service(&self, task_id: FinalTaskId) {
let mut signal = self
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(service) = signal.as_ref() else {
return;
};
if service.sender.try_send(task_id).is_err() && service.sender.is_closed() {
*signal = None;
}
}
fn signal_task_service_cancellation(&self, task_id: FinalTaskId) {
let cancellation_wake = self
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref()
.map(|service| Arc::clone(&service.cancellation_wake));
self.signal_task_service(task_id.clone());
if let Some(cancellation_wake) = cancellation_wake {
cancellation_wake.wake_if_active(&task_id);
}
}
fn register_task_cancellation_wake(
&self,
service_id: u64,
task_id: &FinalTaskId,
) -> McpResult<FinalTaskCancellationWakeRegistration> {
let cancellation_wake = {
let signal = self
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let service = signal.as_ref().ok_or_else(|| {
McpError::internal_error("Task service runner lost its cancellation wake authority")
})?;
if service.service_id != service_id {
return Err(McpError::internal_error(
"A stale task service runner cannot register cancellation wakeups",
));
}
Arc::clone(&service.cancellation_wake)
};
cancellation_wake.activate(task_id)
}
}
impl AuthorizedTaskServiceRunner {
pub fn with_max_concurrent_handoffs(mut self, maximum: usize) -> McpResult<Self> {
if !(1..=MAX_CONCURRENT_TASK_HANDOFFS).contains(&maximum) {
return Err(McpError::invalid_params(
"Task service concurrency must be between 1 and 256",
));
}
self.max_concurrent_handoffs = maximum;
Ok(self)
}
#[must_use]
pub const fn max_concurrent_handoffs(&self) -> usize {
self.max_concurrent_handoffs
}
fn handoff_executor(&self) -> FinalTaskHandoffExecutor {
FinalTaskHandoffExecutor {
runtime: self.runtime.clone(),
service_id: self.service_id,
dispatch_owner: self.dispatch_owner.clone(),
supervisor: Arc::clone(&self.supervisor),
}
}
pub async fn run(mut self, cx: &Cx) -> McpResult<()> {
self.run_service(cx).await
}
pub async fn run_service(&mut self, cx: &Cx) -> McpResult<()> {
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
self.runtime.store.execution_limits()?;
let _readiness_lease = self.runtime.mark_task_service_ready(self.service_id)?;
if self.max_concurrent_handoffs > 1 {
return self.run_concurrent_service(cx).await;
}
if let Err(error) = self.recover_pending(cx).await {
if cx.checkpoint().is_err() {
return Ok(());
}
return Err(error);
}
loop {
let wake = {
let mut receiver = std::pin::pin!(self.receiver.recv(cx));
let mut recovery_wake = std::pin::pin!(asupersync::time::sleep(
cx.now(),
FINAL_TASK_RECOVERY_WAKE_INTERVAL,
));
std::future::poll_fn(|task_context| {
if let std::task::Poll::Ready(result) = receiver.as_mut().poll(task_context) {
return std::task::Poll::Ready(result.map(Some));
}
if recovery_wake.as_mut().poll(task_context).is_ready() {
return std::task::Poll::Ready(Ok(None));
}
std::task::Poll::Pending
})
.await
};
let task_id = match wake {
Ok(Some(task_id)) => Some(task_id),
Ok(None) => None,
Err(_) if cx.checkpoint().is_err() => return Ok(()),
Err(error) => {
return Err(McpError::internal_error(format!(
"Task service wakeup receive failed: {error}"
)));
}
};
if let Some(task_id) = task_id {
if let Err(error) = self.resume_task(cx, &task_id).await {
if cx.checkpoint().is_err() {
return Ok(());
}
return Err(error);
}
}
if let Err(error) = self.recover_pending(cx).await {
if cx.checkpoint().is_err() {
return Ok(());
}
return Err(error);
}
}
}
async fn run_concurrent_service(&mut self, cx: &Cx) -> McpResult<()> {
enum Turn {
Completed(usize, McpResult<()>),
Recover,
Cancelled,
ReceiveFailed(String),
}
let executor = self.handoff_executor();
let mut active: Vec<FinalTaskConcurrentHandoff<'_>> =
Vec::with_capacity(self.max_concurrent_handoffs);
let mut ready_turns = 0;
let mut attempted_initial_cursor = self.initial_recovery_cursor.clone();
let mut attempted_accepted_cursor = self.accepted_recovery_cursor.clone();
loop {
if cx.checkpoint().is_err() {
return Ok(());
}
self.runtime.expire_pending_execution(cx)?;
let mut scanned = 0;
while active.len() < self.max_concurrent_handoffs
&& scanned < MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN
{
let excluded_task_ids = active
.iter()
.map(|handoff| handoff.task_id.clone())
.collect();
let (kind, handoff) =
match self.next_recovery_handoff_excluding(cx, &excluded_task_ids) {
Ok(Some(recovered)) => recovered,
Ok(None) => break,
Err(_) if cx.checkpoint().is_err() => return Ok(()),
Err(error) => return Err(error),
};
scanned += 1;
let task_id = final_task_handoff_task_id(&handoff).clone();
active.push(FinalTaskConcurrentHandoff {
kind,
task_id,
future: executor.resume_handoff(cx, handoff),
wake: Arc::new(FinalTaskHandoffWake {
ready: AtomicBool::new(true),
parent: Mutex::new(None),
}),
});
}
let scan_has_more_capacity = scanned == MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN
&& active.len() < self.max_concurrent_handoffs;
let has_capacity = active.len() < self.max_concurrent_handoffs;
let turn = {
let mut receiver = std::pin::pin!(self.receiver.recv(cx));
let mut recovery_wake = std::pin::pin!(asupersync::time::sleep(
cx.now(),
FINAL_TASK_RECOVERY_WAKE_INTERVAL,
));
std::future::poll_fn(|task_context| {
if cx.checkpoint().is_err() {
return std::task::Poll::Ready(Turn::Cancelled);
}
for (index, handoff) in active.iter_mut().enumerate() {
*handoff
.wake
.parent
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(task_context.waker().clone());
if !handoff.wake.ready.swap(false, TaskServiceOrdering::AcqRel) {
continue;
}
let waker = std::task::Waker::from(Arc::clone(&handoff.wake));
let mut handoff_context = std::task::Context::from_waker(&waker);
match handoff.kind {
FinalTaskRecoveryKind::Initial => {
attempted_initial_cursor = Some(handoff.task_id.clone());
}
FinalTaskRecoveryKind::Resumed => {
attempted_accepted_cursor = Some(handoff.task_id.clone());
}
}
if let std::task::Poll::Ready(result) =
handoff.future.as_mut().poll(&mut handoff_context)
{
return std::task::Poll::Ready(Turn::Completed(index, result));
}
if cx.checkpoint().is_err() {
return std::task::Poll::Ready(Turn::Cancelled);
}
}
if scan_has_more_capacity {
return std::task::Poll::Ready(Turn::Recover);
}
if has_capacity {
if let std::task::Poll::Ready(result) = receiver.as_mut().poll(task_context)
{
return std::task::Poll::Ready(match result {
Ok(_) => Turn::Recover,
Err(_) if cx.checkpoint().is_err() => Turn::Cancelled,
Err(error) => Turn::ReceiveFailed(error.to_string()),
});
}
}
if recovery_wake.as_mut().poll(task_context).is_ready() {
return std::task::Poll::Ready(Turn::Recover);
}
std::task::Poll::Pending
})
.await
};
match turn {
Turn::Completed(index, result) => {
let completed = active.remove(index);
if let Err(error) = result {
if cx.checkpoint().is_err() {
return Ok(());
}
self.initial_recovery_cursor = attempted_initial_cursor;
self.accepted_recovery_cursor = attempted_accepted_cursor;
self.next_recovery_kind = completed.kind.other();
return Err(error);
}
drop(completed);
}
Turn::Recover => {}
Turn::Cancelled => return Ok(()),
Turn::ReceiveFailed(error) => {
return Err(McpError::internal_error(format!(
"Task service wakeup receive failed: {error}"
)));
}
}
ready_turns += 1;
if ready_turns == MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN {
asupersync::runtime::yield_now().await;
ready_turns = 0;
}
}
}
fn next_recovery_handoff(
&mut self,
cx: &Cx,
) -> McpResult<Option<(FinalTaskRecoveryKind, FinalTaskSupervisorHandoff)>> {
self.next_recovery_handoff_excluding(cx, &BTreeSet::new())
}
fn next_recovery_handoff_excluding(
&mut self,
cx: &Cx,
excluded_task_ids: &BTreeSet<FinalTaskId>,
) -> McpResult<Option<(FinalTaskRecoveryKind, FinalTaskSupervisorHandoff)>> {
self.runtime.expire_pending_execution(cx)?;
let recovered = match self.next_recovery_kind {
FinalTaskRecoveryKind::Initial => {
if let Some(handoff) = self.runtime.recover_initial_work_excluding(
cx,
&self.dispatch_owner,
&mut self.initial_recovery_cursor,
excluded_task_ids,
)? {
Some((
FinalTaskRecoveryKind::Initial,
FinalTaskSupervisorHandoff::Initial(handoff),
))
} else {
self.runtime
.recover_accepted_input_excluding(
cx,
&self.dispatch_owner,
&mut self.accepted_recovery_cursor,
excluded_task_ids,
)?
.map(|handoff| {
(
FinalTaskRecoveryKind::Resumed,
FinalTaskSupervisorHandoff::Resumed(handoff),
)
})
}
}
FinalTaskRecoveryKind::Resumed => {
if let Some(handoff) = self.runtime.recover_accepted_input_excluding(
cx,
&self.dispatch_owner,
&mut self.accepted_recovery_cursor,
excluded_task_ids,
)? {
Some((
FinalTaskRecoveryKind::Resumed,
FinalTaskSupervisorHandoff::Resumed(handoff),
))
} else {
self.runtime
.recover_initial_work_excluding(
cx,
&self.dispatch_owner,
&mut self.initial_recovery_cursor,
excluded_task_ids,
)?
.map(|handoff| {
(
FinalTaskRecoveryKind::Initial,
FinalTaskSupervisorHandoff::Initial(handoff),
)
})
}
}
};
if let Some((kind, handoff)) = &recovered {
let task_id = final_task_handoff_task_id(handoff).clone();
match kind {
FinalTaskRecoveryKind::Initial => self.initial_recovery_cursor = Some(task_id),
FinalTaskRecoveryKind::Resumed => self.accepted_recovery_cursor = Some(task_id),
}
self.next_recovery_kind = kind.other();
}
Ok(recovered)
}
async fn recover_pending(&mut self, cx: &Cx) -> McpResult<()> {
let mut last_recovered_task_id = None;
let mut first_retryable_error = None;
let mut retried_handoffs = BTreeSet::new();
for _ in 0..MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN {
let Some((kind, handoff)) = self.next_recovery_handoff(cx)? else {
break;
};
let task_id = final_task_handoff_task_id(&handoff).clone();
let retry_key = (kind, task_id.clone());
last_recovered_task_id = Some(task_id);
if let Err(error) = self.resume_handoff(cx, handoff).await {
if !retried_handoffs.insert(retry_key) {
if first_retryable_error.is_none() {
first_retryable_error = Some(error);
}
break;
}
if first_retryable_error.is_none() {
first_retryable_error = Some(error);
}
}
}
if let Some(task_id) = last_recovered_task_id {
self.runtime.signal_task_service(task_id);
}
if let Some(error) = first_retryable_error {
Err(error)
} else {
Ok(())
}
}
async fn resume_task(&self, cx: &Cx, task_id: &FinalTaskId) -> McpResult<()> {
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
if let Some(initial) =
self.runtime
.take_initial_work_with_checkpoint(cx, task_id, &self.dispatch_owner)?
{
self.resume_handoff(cx, FinalTaskSupervisorHandoff::Initial(initial))
.await?;
return Ok(());
}
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
if let Some(accepted) =
self.runtime
.take_accepted_input_with_checkpoint(cx, task_id, &self.dispatch_owner)?
{
self.resume_handoff(cx, FinalTaskSupervisorHandoff::Resumed(accepted))
.await?;
}
Ok(())
}
fn resume_handoff<'a>(
&self,
cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
self.handoff_executor().resume_handoff(cx, handoff)
}
}
impl FinalTaskHandoffExecutor {
fn resume_handoff<'a>(
&self,
cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let executor = self.clone();
let guard = FinalTaskExecutionGuard::new(&self.runtime, &self.dispatch_owner, &handoff);
Box::pin(async move { executor.execute_handoff(cx, handoff, guard).await })
}
async fn execute_handoff(
&self,
cx: &Cx,
mut handoff: FinalTaskSupervisorHandoff,
mut guard: FinalTaskExecutionGuard,
) -> McpResult<()> {
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
if !guard.elect()? {
guard.disarm();
return Ok(());
}
if guard.retire_if_execution_expired()? {
return Ok(());
}
let cancellation_wake = self
.runtime
.register_task_cancellation_wake(self.service_id, guard.task_id())?;
if guard.is_authoritatively_expired() {
guard.disarm();
return Ok(());
}
if guard.retire_if_cancellation_requested()? {
return Ok(());
}
handoff.attach_authority(guard.authority()?);
match self
.run_supervisor_with_lease_heartbeat(cx, handoff, &mut guard, &cancellation_wake)
.await
{
Ok(()) => {
if guard.retire_if_execution_expired()? {
return Ok(());
}
if guard.is_authoritatively_expired() {
guard.disarm();
return Ok(());
}
if guard.retire_if_cancellation_requested()? {
return Ok(());
}
if guard.is_recoverable_without_transition()? {
let restored = guard.restore()?;
if !restored {
if guard.retire_if_cancellation_requested()? {
return Ok(());
}
return Err(McpError::internal_error(
"Final task supervisor returned success without a fenced transition and its handoff could not be restored",
));
}
return Err(McpError::internal_error(
"Final task supervisor returned success without a fenced task transition",
));
}
let finished = guard.finish()?;
if !finished && guard.retire_if_cancellation_requested()? {
return Ok(());
}
guard.disarm();
Ok(())
}
Err(error) => {
if guard.retire_if_execution_expired()? {
return Ok(());
}
if guard.is_authoritatively_expired() {
guard.disarm();
return Err(error);
}
if guard.retire_if_cancellation_requested()? {
return Ok(());
}
let restored = guard.restore()?;
if !restored && guard.retire_if_cancellation_requested()? {
return Ok(());
}
Err(error)
}
}
}
async fn run_supervisor_with_lease_heartbeat(
&self,
cx: &Cx,
handoff: FinalTaskSupervisorHandoff,
guard: &mut FinalTaskExecutionGuard,
cancellation_wake: &FinalTaskCancellationWakeRegistration,
) -> McpResult<()> {
if guard.retire_if_execution_expired()? {
return Ok(());
}
guard.ensure_live_dispatch_for_application()?;
let mut supervisor = self.supervisor.resume(cx, handoff);
loop {
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
self.runtime.expire_pending_execution(cx)?;
if guard.retire_if_execution_expired()? {
return Ok(());
}
if guard.is_authoritatively_expired() {
guard.disarm();
return Ok(());
}
if guard.retire_if_cancellation_requested()? {
return Ok(());
}
let heartbeat_interval = guard.bounded_heartbeat_interval()?;
let mut heartbeat = Box::pin(asupersync::time::sleep(cx.now(), heartbeat_interval));
let mut cancellation_boundary_granted = false;
let completed = std::future::poll_fn(|task_context| {
if let Err(error) = cx.checkpoint() {
return std::task::Poll::Ready(Some(Err(McpError::internal_error(
error.to_string(),
))));
}
match guard.retire_if_execution_expired() {
Ok(true) => return std::task::Poll::Ready(Some(Ok(()))),
Ok(false) => {}
Err(error) => return std::task::Poll::Ready(Some(Err(error))),
}
if guard.is_authoritatively_expired() {
return std::task::Poll::Ready(None);
}
if let Err(error) = guard.ensure_live_dispatch_for_application() {
return std::task::Poll::Ready(Some(Err(error)));
}
cancellation_wake.register_waker(task_context.waker());
if let std::task::Poll::Ready(result) = supervisor.as_mut().poll(task_context) {
match guard.retire_if_execution_expired() {
Ok(true) => return std::task::Poll::Ready(Some(Ok(()))),
Ok(false) => {}
Err(error) => return std::task::Poll::Ready(Some(Err(error))),
}
return std::task::Poll::Ready(Some(result));
}
match guard.retire_if_execution_expired() {
Ok(true) => return std::task::Poll::Ready(Some(Ok(()))),
Ok(false) => {}
Err(error) => return std::task::Poll::Ready(Some(Err(error))),
}
if guard.is_authoritatively_expired() {
return std::task::Poll::Ready(None);
}
match guard.is_cancellation_requested() {
Ok(true) => {
if cancellation_boundary_granted {
return std::task::Poll::Ready(Some(Ok(())));
}
cancellation_boundary_granted = true;
task_context.waker().wake_by_ref();
return std::task::Poll::Pending;
}
Ok(false) => {}
Err(error) => return std::task::Poll::Ready(Some(Err(error))),
}
if heartbeat.as_mut().poll(task_context).is_ready() {
return std::task::Poll::Ready(None);
}
std::task::Poll::Pending
})
.await;
let Some(result) = completed else {
if guard.retire_if_execution_expired()? {
return Ok(());
}
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
if guard.is_authoritatively_expired() {
guard.disarm();
return Ok(());
}
if guard.retire_if_cancellation_requested()? {
return Ok(());
}
if !guard.renew()? {
if guard.retire_if_execution_expired()? {
return Ok(());
}
if guard.is_authoritatively_expired() {
guard.disarm();
return Ok(());
}
return Err(McpError::internal_error(
"Final task dispatch lease was lost while application work was running",
));
}
continue;
};
if let Err(error) = result {
if guard.is_authoritatively_expired() {
guard.disarm();
}
return Err(error);
}
if guard.is_authoritatively_expired() {
guard.disarm();
return Ok(());
}
if guard.retire_if_cancellation_requested()? {
return Ok(());
}
return result;
}
}
}
pub(crate) const HOSTED_TASK_SERVICE_QUEUE_CAPACITY: usize = 64;
pub(crate) const HOSTED_TASK_SERVICE_STARTUP_BOUND: StdDuration = StdDuration::from_secs(2);
pub(crate) const HOSTED_TASK_SERVICE_SETTLEMENT_BOUND: StdDuration = StdDuration::from_secs(4);
const HOSTED_TASK_SERVICE_POLL: StdDuration = StdDuration::from_millis(1);
type HostedRunnerSlot = Arc<Mutex<Option<AuthorizedTaskServiceRunner>>>;
#[derive(Clone)]
pub(crate) struct TaskServiceHost {
runtime: FinalTaskRuntime,
slot: HostedRunnerSlot,
}
#[must_use = "a hosted Task service must be settled before its serve returns"]
pub(crate) struct HostedTaskService {
runtime: FinalTaskRuntime,
handle: asupersync::runtime::TaskHandle<McpResult<()>>,
failure_signal: Arc<AtomicBool>,
}
struct ReturnRunnerOnDrop {
slot: HostedRunnerSlot,
runner: Option<AuthorizedTaskServiceRunner>,
}
struct HostedServiceExitSignal {
cx: Cx,
failed: Arc<AtomicBool>,
}
impl Drop for HostedServiceExitSignal {
fn drop(&mut self) {
if !self.cx.is_cancel_requested() {
self.failed.store(true, TaskServiceOrdering::Release);
}
}
}
impl Drop for ReturnRunnerOnDrop {
fn drop(&mut self) {
let runner = self.runner.take();
*self
.slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = runner;
}
}
impl TaskServiceHost {
pub(crate) fn install(
runtime: &FinalTaskRuntime,
supervisor: Arc<dyn ApplicationTaskSupervisor>,
max_concurrent_handoffs: usize,
) -> McpResult<Self> {
if !(1..=MAX_CONCURRENT_TASK_HANDOFFS).contains(&max_concurrent_handoffs) {
return Err(McpError::invalid_params(
"Task service concurrency must be between 1 and 256",
));
}
let runner = runtime
.install_task_service(HOSTED_TASK_SERVICE_QUEUE_CAPACITY, supervisor)?
.with_max_concurrent_handoffs(max_concurrent_handoffs)?;
Ok(Self {
runtime: runtime.clone(),
slot: Arc::new(Mutex::new(Some(runner))),
})
}
pub(crate) fn start(&self, cx: &Cx) -> McpResult<HostedTaskService> {
cx.checkpoint().map_err(|_| McpError::request_cancelled())?;
if cx.timer_driver().is_none() {
return Err(McpError::internal_error(
"A hosted Task service requires the caller's timer driver",
));
}
let runner = self
.slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
.ok_or_else(|| {
McpError::internal_error("The hosted Task service is already running")
})?;
let mut guard = ReturnRunnerOnDrop {
slot: Arc::clone(&self.slot),
runner: Some(runner),
};
let failure_signal = Arc::new(AtomicBool::new(false));
let service_failure = Arc::clone(&failure_signal);
let handle = cx
.spawn(move |service_cx| async move {
let _exit_signal = HostedServiceExitSignal {
cx: service_cx.clone(),
failed: service_failure,
};
let Some(runner) = guard.runner.as_mut() else {
return Ok(());
};
runner.run_service(&service_cx).await
})
.map_err(|error| {
McpError::internal_error(format!("Hosted Task service admission failed: {error}"))
})?;
Ok(HostedTaskService {
runtime: self.runtime.clone(),
handle,
failure_signal,
})
}
pub(crate) async fn start_ready(&self, cx: &Cx) -> McpResult<HostedTaskService> {
let mut hosted = self.start(cx)?;
match hosted.ready(cx).await {
Ok(()) => Ok(hosted),
Err(error) => {
let _ = hosted.settle(cx).await;
Err(error)
}
}
}
pub(crate) fn start_ready_blocking(&self, cx: &Cx) -> McpResult<HostedTaskService> {
let mut hosted = self.start(cx)?;
match hosted.ready_blocking() {
Ok(()) => Ok(hosted),
Err(error) => {
let _ = hosted.settle_blocking();
Err(error)
}
}
}
}
impl HostedTaskService {
pub(crate) fn failure_signal(&self) -> Arc<AtomicBool> {
Arc::clone(&self.failure_signal)
}
pub(crate) fn check_running(&self) -> McpResult<()> {
if self.handle.is_finished() {
Err(McpError::internal_error(
"The hosted Task service stopped while serving",
))
} else {
Ok(())
}
}
fn not_ready(&mut self) -> McpResult<bool> {
if self.runtime.is_task_service_ready() {
return Ok(false);
}
if self.handle.is_finished() {
return Err(McpError::internal_error(
"The hosted Task service stopped before publishing readiness",
));
}
Ok(true)
}
async fn ready(&mut self, cx: &Cx) -> McpResult<()> {
let deadline = cx.now() + HOSTED_TASK_SERVICE_STARTUP_BOUND;
while self.not_ready()? {
cx.checkpoint().map_err(|_| McpError::request_cancelled())?;
asupersync::time::timeout_at(
deadline,
asupersync::time::sleep(cx.now(), HOSTED_TASK_SERVICE_POLL),
)
.await
.map_err(|_| hosted_task_service_not_ready())?;
}
Ok(())
}
fn ready_blocking(&mut self) -> McpResult<()> {
let deadline = Instant::now() + HOSTED_TASK_SERVICE_STARTUP_BOUND;
while self.not_ready()? {
if Instant::now() >= deadline {
return Err(hosted_task_service_not_ready());
}
std::thread::sleep(HOSTED_TASK_SERVICE_POLL);
}
Ok(())
}
pub(crate) async fn settle(mut self, cx: &Cx) -> McpResult<()> {
self.handle.abort();
let deadline = cx.now() + HOSTED_TASK_SERVICE_SETTLEMENT_BOUND;
let mut exceeded_bound = false;
loop {
exceeded_bound |= cx.now() >= deadline;
let joined = match self.handle.try_join() {
Ok(Some(result)) => Some(result),
Ok(None) => None,
Err(error) => Some(hosted_task_service_exit(Err(error))),
};
if let Some(result) = joined {
return if exceeded_bound {
Err(hosted_task_service_unsettled())
} else {
result
};
}
asupersync::runtime::yield_now().await;
asupersync::time::sleep(cx.now(), HOSTED_TASK_SERVICE_POLL).await;
}
}
pub(crate) fn settle_blocking(mut self) -> McpResult<()> {
self.handle.abort();
let deadline = Instant::now() + HOSTED_TASK_SERVICE_SETTLEMENT_BOUND;
let mut exceeded_bound = false;
loop {
exceeded_bound |= Instant::now() >= deadline;
let joined = match self.handle.try_join() {
Ok(Some(result)) => Some(result),
Ok(None) => None,
Err(error) => Some(hosted_task_service_exit(Err(error))),
};
if let Some(result) = joined {
return if exceeded_bound {
Err(hosted_task_service_unsettled())
} else {
result
};
}
std::thread::sleep(HOSTED_TASK_SERVICE_POLL);
}
}
}
impl Drop for HostedTaskService {
fn drop(&mut self) {
self.handle.abort();
}
}
fn hosted_task_service_exit(
joined: Result<McpResult<()>, asupersync::runtime::JoinError>,
) -> McpResult<()> {
match joined {
Ok(result) => result,
Err(asupersync::runtime::JoinError::Cancelled(_)) => Ok(()),
Err(error) => Err(McpError::internal_error(format!(
"Hosted Task service join failed: {error:?}"
))),
}
}
fn hosted_task_service_not_ready() -> McpError {
McpError::internal_error("The hosted Task service did not become ready within its bound")
}
fn hosted_task_service_unsettled() -> McpError {
McpError::internal_error("The hosted Task service did not settle within its bound")
}
fn final_task_handoff_task_id(handoff: &FinalTaskSupervisorHandoff) -> &FinalTaskId {
match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => initial.task_id(),
FinalTaskSupervisorHandoff::Resumed(accepted) => accepted.task_id(),
}
}
enum FinalTaskHandoffRestoration {
Initial(FinalTaskWorkDescriptor),
Resumed(FinalTaskInputResponses),
}
struct FinalTaskExecutionGuard {
runtime: FinalTaskRuntime,
task_id: FinalTaskId,
generation: u64,
owner_id: String,
dispatch_fence: Option<u64>,
transition_committed: Arc<AtomicBool>,
retention_deadline: Option<FinalTaskRetentionDeadline>,
execution_budget: Option<FinalTaskExecutionBudget>,
restoration: Option<FinalTaskHandoffRestoration>,
}
impl FinalTaskExecutionGuard {
fn new(
runtime: &FinalTaskRuntime,
owner_id: &str,
handoff: &FinalTaskSupervisorHandoff,
) -> Self {
let (task_id, generation, restoration) = match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => (
initial.task_id().clone(),
initial.generation(),
FinalTaskHandoffRestoration::Initial(initial.restore_copy()),
),
FinalTaskSupervisorHandoff::Resumed(accepted) => (
accepted.task_id().clone(),
accepted.generation(),
FinalTaskHandoffRestoration::Resumed(accepted.restore_copy()),
),
};
Self {
runtime: runtime.clone(),
task_id,
generation,
owner_id: owner_id.to_owned(),
dispatch_fence: None,
transition_committed: Arc::new(AtomicBool::new(false)),
retention_deadline: None,
execution_budget: None,
restoration: Some(restoration),
}
}
fn elect(&mut self) -> McpResult<bool> {
let Some(current) = self.runtime.load_optional_task_snapshot(&self.task_id)? else {
return Ok(false);
};
if current.generation() != self.generation {
return Ok(false);
}
let Some(budget) = self
.runtime
.store
.task_execution_budget_if_current(¤t)?
else {
return Ok(false);
};
if budget.attempt_deadline.is_none() || budget.attempts == 0 {
return Err(McpError::internal_error(
"Final task store did not bound the claimed execution attempt",
));
}
self.execution_budget = Some(budget);
let Some(deadline) = self
.runtime
.task_retention_deadline(&self.task_id, self.generation)?
else {
return Ok(false);
};
let Some(dispatch_fence) =
self.runtime
.begin_handoff_dispatch(&self.task_id, self.generation, &self.owner_id)?
else {
return Ok(false);
};
self.retention_deadline = Some(deadline);
self.dispatch_fence = Some(dispatch_fence);
Ok(true)
}
fn task_id(&self) -> &FinalTaskId {
&self.task_id
}
fn retire_if_execution_expired(&mut self) -> McpResult<bool> {
let Some(budget) = self.execution_budget else {
return Ok(false);
};
let deadline = budget.attempt_deadline.unwrap_or(budget.total_deadline);
if self.runtime.retention_clock_now() < deadline {
return Ok(false);
}
let _ = self.runtime.load_optional_task_snapshot(&self.task_id)?;
self.disarm();
Ok(true)
}
fn is_authoritatively_expired(&self) -> bool {
let Some(FinalTaskRetentionDeadline::Finite(deadline)) = self.retention_deadline else {
return false;
};
let store_now = self.runtime.retention_clock_now();
store_now >= deadline
}
fn is_cancellation_requested(&self) -> McpResult<bool> {
if self.is_authoritatively_expired() {
return Ok(false);
}
let Some(current) = self.runtime.load_optional_task_snapshot(&self.task_id)? else {
if self.is_authoritatively_expired() {
return Ok(false);
}
return Err(McpError::invalid_params("Task not found"));
};
if current.generation() != self.generation {
return Ok(false);
}
self.runtime.store.is_cancellation_requested(&self.task_id)
}
fn retire_if_cancellation_requested(&mut self) -> McpResult<bool> {
if self.is_authoritatively_expired() {
return Ok(false);
}
if !self.is_cancellation_requested()? {
return Ok(false);
}
self.runtime.fenced_honor_cancellation(
&self.task_id,
self.generation,
&self.owner_id,
self.dispatch_fence.ok_or_else(|| {
McpError::internal_error(
"Final task cancellation retirement requires an elected dispatch fence",
)
})?,
None,
)?;
self.disarm();
Ok(true)
}
fn renew(&self) -> McpResult<bool> {
if self.transition_committed.load(TaskServiceOrdering::Acquire) {
return Ok(true);
}
let Some(dispatch_fence) = self.dispatch_fence else {
return Ok(false);
};
self.runtime.renew_handoff_dispatch(
&self.task_id,
self.generation,
&self.owner_id,
dispatch_fence,
)
}
fn ensure_live_dispatch_for_application(&self) -> McpResult<()> {
if self.renew()? {
Ok(())
} else {
Err(McpError::internal_error(
"Final task dispatch lease was lost while application work was running",
))
}
}
fn heartbeat_interval(&self) -> McpResult<StdDuration> {
self.runtime.handoff_dispatch_lease_heartbeat_interval()
}
fn bounded_heartbeat_interval(&self) -> McpResult<StdDuration> {
let mut base_interval = self
.heartbeat_interval()?
.min(FINAL_TASK_RECOVERY_WAKE_INTERVAL);
if let Some(budget) = self.execution_budget {
let deadline = budget.attempt_deadline.unwrap_or(budget.total_deadline);
base_interval = base_interval
.min(deadline.saturating_duration_since(self.runtime.retention_clock_now()));
}
if let Some(FinalTaskRetentionDeadline::Finite(deadline)) = self.retention_deadline {
let store_now = self.runtime.retention_clock_now();
if store_now >= deadline {
return Ok(StdDuration::ZERO);
}
let remaining = deadline.saturating_duration_since(store_now);
return Ok(base_interval.min(remaining));
}
Ok(base_interval)
}
fn is_recoverable_without_transition(&self) -> McpResult<bool> {
if self.is_authoritatively_expired() {
return Ok(false);
}
let Some(current) = self.runtime.load_optional_task_snapshot(&self.task_id)? else {
if self.is_authoritatively_expired() {
return Ok(false);
}
return Err(McpError::invalid_params("Task not found"));
};
Ok(current.generation() == self.generation
&& matches!(current.task(), FinalTask::Working(_))
&& !self
.runtime
.store
.is_cancellation_requested(&self.task_id)?)
}
fn authority(&self) -> McpResult<FinalTaskHandoffAuthority> {
let dispatch_fence = self.dispatch_fence.ok_or_else(|| {
McpError::internal_error(
"Final task handoff authority was requested before dispatch election",
)
})?;
Ok(FinalTaskHandoffAuthority {
runtime: self.runtime.clone(),
task_id: self.task_id.clone(),
generation: self.generation,
owner_id: self.owner_id.clone(),
dispatch_fence,
transition_committed: Arc::clone(&self.transition_committed),
execution_budget: self.execution_budget.ok_or_else(|| {
McpError::internal_error("Final task handoff has no execution budget")
})?,
})
}
fn finish(&self) -> McpResult<bool> {
let Some(dispatch_fence) = self.dispatch_fence else {
return Ok(false);
};
self.runtime.finish_handoff_dispatch(
&self.task_id,
self.generation,
&self.owner_id,
dispatch_fence,
)
}
fn disarm(&mut self) {
self.restoration = None;
}
fn restore(&mut self) -> McpResult<bool> {
if self.is_authoritatively_expired() {
self.disarm();
return Ok(false);
}
let Some(restoration) = self.restoration.as_ref() else {
return Ok(false);
};
let restored = match restoration {
FinalTaskHandoffRestoration::Initial(work_descriptor) => {
self.runtime.restore_initial_work(
&self.task_id,
self.generation,
&self.owner_id,
self.dispatch_fence,
work_descriptor.clone(),
)
}
FinalTaskHandoffRestoration::Resumed(input_responses) => {
self.runtime.restore_accepted_input(
&self.task_id,
self.generation,
&self.owner_id,
self.dispatch_fence,
input_responses.clone(),
)
}
};
if matches!(&restored, Ok(true)) {
self.restoration = None;
}
restored
}
}
impl Drop for FinalTaskExecutionGuard {
fn drop(&mut self) {
if self.retire_if_execution_expired().unwrap_or(false) {
return;
}
if self.is_authoritatively_expired() {
self.disarm();
return;
}
if self.retire_if_cancellation_requested().unwrap_or(false) {
return;
}
let _ = self.restore();
let _ = self.retire_if_cancellation_requested();
}
}
impl Drop for FinalTaskServiceReadinessLease {
fn drop(&mut self) {
let mut signal = self
.runtime
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(service) = signal.as_mut() else {
return;
};
if service.service_id == self.service_id
&& service.ready_generation == Some(self.ready_generation)
{
service.ready_generation = None;
}
}
}
impl Drop for AuthorizedTaskServiceRunner {
fn drop(&mut self) {
self.receiver.close();
let mut signal = self
.runtime
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if signal
.as_ref()
.is_some_and(|service| service.service_id == self.service_id)
{
*signal = None;
}
}
}
pub(crate) fn dispatch_final_tasks_get(
runtime: &FinalTaskRuntime,
ctx: &McpContext,
parameters: serde_json::Value,
) -> McpResult<serde_json::Value> {
let parameters = serde_json::from_value::<FinalGetTaskParams>(parameters)
.map_err(|_| McpError::invalid_params("Invalid final tasks/get parameters"))?;
validate_final_task_request_meta(¶meters.request, "tasks/get")?;
let task = runtime
.task_for_request(ctx, ¶meters.task_id)?
.into_task();
serde_json::to_value(fastmcp_protocol::CompleteTaskResult {
task,
meta: None,
additional: BTreeMap::new(),
})
.map_err(|_| McpError::internal_error("final tasks/get response serialization failed"))
}
pub(crate) fn dispatch_final_tasks_update(
runtime: &FinalTaskRuntime,
ctx: &McpContext,
parameters: serde_json::Value,
) -> McpResult<serde_json::Value> {
let parameters = serde_json::from_value::<UpdateTaskParams>(parameters)
.map_err(|_| McpError::invalid_params("Invalid final tasks/update parameters"))?;
validate_final_task_request_meta(¶meters.request, "tasks/update")?;
let current = runtime.task_for_request(ctx, ¶meters.task_id)?;
serde_json::to_value(runtime.update_task_snapshot(¤t, ¶meters.input_responses)?)
.map_err(|_| McpError::internal_error("final tasks/update response serialization failed"))
}
pub(crate) fn dispatch_final_tasks_cancel(
runtime: &FinalTaskRuntime,
ctx: &McpContext,
parameters: serde_json::Value,
) -> McpResult<serde_json::Value> {
let parameters = serde_json::from_value::<FinalCancelTaskParams>(parameters)
.map_err(|_| McpError::invalid_params("Invalid final tasks/cancel parameters"))?;
validate_final_task_request_meta(¶meters.request, "tasks/cancel")?;
let current = runtime.task_for_request(ctx, ¶meters.task_id)?;
serde_json::to_value(runtime.cancel_task_snapshot(¤t)?)
.map_err(|_| McpError::internal_error("final tasks/cancel response serialization failed"))
}
fn task_request_principal(ctx: &McpContext) -> McpResult<Option<Sha256Digest>> {
ctx.auth()
.as_ref()
.map(|auth| crate::auth::principal_fingerprint(Some(auth)))
.transpose()
}
fn validate_final_task_request_meta(
request: &FinalTaskRequestMeta,
method: &'static str,
) -> McpResult<()> {
let Ok(protocol_version) = request.meta.protocol_version() else {
return Err(McpError::invalid_params(format!(
"Invalid final {method} parameters"
)));
};
let client_capabilities = request.meta.client_capabilities().ok().flatten();
if protocol_version.is_some_and(|version| version != FINAL_PROTOCOL_VERSION)
|| client_capabilities.is_none()
{
return Err(McpError::invalid_params(format!(
"Invalid final {method} parameters"
)));
}
Ok(())
}
fn generate_final_task_id() -> McpResult<FinalTaskId> {
let identifier = draw_security_identifier().map_err(|error| {
McpError::internal_error(format!("Task identifier generation failed: {error}"))
})?;
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(identifier.as_bytes());
FinalTaskId::parse(encoded).map_err(|error| McpError::internal_error(error.to_string()))
}
#[cfg(not(test))]
fn generate_final_task_dispatch_owner() -> McpResult<String> {
generate_final_task_id().map(|task_id| task_id.as_str().to_owned())
}
fn final_task_timestamp() -> McpResult<FinalTaskTimestamp> {
FinalTaskTimestamp::parse(
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true),
)
.map_err(|error| McpError::internal_error(error.to_string()))
}
fn final_task_duration(milliseconds: u64) -> McpResult<FinalTaskDuration> {
serde_json::from_value(serde_json::json!(milliseconds))
.map_err(|error| McpError::invalid_params(format!("invalid task duration: {error}")))
}
fn transition_final_task_base(
mut base: FinalTaskBase,
status: FinalTaskStatus,
status_message: Option<String>,
) -> McpResult<FinalTaskBase> {
base.status = status;
base.status_message = status_message;
base.last_updated_at = final_task_timestamp()?;
Ok(base)
}
pub(crate) fn transition_terminal_final_task_base(
base: FinalTaskBase,
status: FinalTaskStatus,
status_message: Option<String>,
) -> McpResult<FinalTaskBase> {
if !matches!(
status,
FinalTaskStatus::Completed | FinalTaskStatus::Failed | FinalTaskStatus::Cancelled
) {
return Err(McpError::internal_error(
"terminal task transition requires a terminal status",
));
}
transition_final_task_base(base, status, status_message)
}
pub(crate) fn final_task_notification(task: &FinalTask) -> FinalTaskStatusNotification {
FinalTaskStatusNotification::new(FinalTaskStatusNotificationParams {
task: task.clone(),
meta: None,
additional: BTreeMap::new(),
})
}
#[cfg(test)]
impl Default for TaskManager {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
impl std::fmt::Debug for TaskManager {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let task_count = self
.tasks
.read()
.map(|g| g.len())
.unwrap_or_else(|poisoned| poisoned.into_inner().len());
let handler_count = self
.handlers
.read()
.map(|g| g.len())
.unwrap_or_else(|poisoned| poisoned.into_inner().len());
f.debug_struct("TaskManager")
.field("task_count", &task_count)
.field("handler_count", &handler_count)
.field("task_counter", &self.task_counter.load(Ordering::SeqCst))
.field(
"list_changed_notifications",
&self.list_changed_notifications,
)
.field("auto_execute", &self.auto_execute)
.finish_non_exhaustive()
}
}
#[cfg(test)]
pub type SharedTaskManager = Arc<TaskManager>;
#[cfg(test)]
mod tests {
macro_rules! delegate_final_execution_policy {
() => {
fn execution_limits(&self) -> McpResult<FinalTaskExecutionLimits> {
self.inner.execution_limits()
}
fn task_execution_budget_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskExecutionBudget>> {
self.inner.task_execution_budget_if_current(expected)
}
fn expire_task_execution_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.expire_task_execution_if_current(expected)
}
fn next_expired_execution_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_expired_execution_snapshot()
}
};
}
use super::*;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering};
use std::thread;
use std::time::Duration;
fn test_take_input(
store: &InMemoryFinalTaskStore,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskInputResponses>> {
FinalTaskStore::take_input_for_owner_if_current(
store,
expected,
FINAL_TASK_TEST_DIRECT_OWNER,
)
}
fn test_take_initial_work(
store: &InMemoryFinalTaskStore,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
FinalTaskStore::take_initial_work_for_owner_if_current(
store,
expected,
FINAL_TASK_TEST_DIRECT_OWNER,
)
}
fn test_next_initial_work(
store: &InMemoryFinalTaskStore,
) -> McpResult<Option<FinalTaskSnapshot>> {
FinalTaskStore::next_initial_work_snapshot_after(store, None)
}
fn test_next_accepted_input(
store: &InMemoryFinalTaskStore,
) -> McpResult<Option<FinalTaskSnapshot>> {
FinalTaskStore::next_accepted_input_snapshot_after(store, None)
}
fn test_restore_initial_work(
store: &InMemoryFinalTaskStore,
task_id: &FinalTaskId,
generation: u64,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
FinalTaskStore::restore_initial_work_for_owner_if_current(
store,
task_id,
generation,
FINAL_TASK_TEST_DIRECT_OWNER,
None,
work_descriptor,
)
}
fn final_task_runtime(
store: Arc<InMemoryFinalTaskStore>,
delivery_after_durable_commit: Arc<AtomicBool>,
) -> FinalTaskRuntime {
let store_for_emitter = Arc::clone(&store);
FinalTaskRuntime::new(
store,
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(move |notification| {
if store_for_emitter
.get_task(¬ification.params.task.base().task_id)
.expect("in-memory final task store read")
.is_some()
{
delivery_after_durable_commit.store(true, AtomicOrdering::SeqCst);
}
}),
)
}
#[derive(Default)]
struct ConcurrentTaskProbeState {
started: Vec<FinalTaskId>,
live: BTreeSet<FinalTaskId>,
peak_live: usize,
polls: BTreeMap<FinalTaskId, usize>,
actions: BTreeMap<FinalTaskId, bool>,
wakers: BTreeMap<FinalTaskId, std::task::Waker>,
}
#[derive(Default)]
struct ConcurrentTaskProbe {
state: Mutex<ConcurrentTaskProbeState>,
}
impl ConcurrentTaskProbe {
fn release(&self, task_id: &FinalTaskId, fail: bool) {
let waker = {
let mut state = self.state.lock().unwrap();
state.actions.insert(task_id.clone(), fail);
state.wakers.get(task_id).cloned()
};
if let Some(waker) = waker {
waker.wake();
}
}
}
struct ConcurrentTaskProbeLease<'a> {
probe: &'a ConcurrentTaskProbe,
task_id: FinalTaskId,
}
impl Drop for ConcurrentTaskProbeLease<'_> {
fn drop(&mut self) {
let mut state = self.probe.state.lock().unwrap();
assert!(state.live.remove(&self.task_id));
state.wakers.remove(&self.task_id);
}
}
impl ApplicationTaskSupervisor for ConcurrentTaskProbe {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
let task_id = final_task_handoff_task_id(&handoff).clone();
{
let mut state = self.state.lock().unwrap();
assert!(state.live.insert(task_id.clone()));
state.started.push(task_id.clone());
state.peak_live = state.peak_live.max(state.live.len());
}
let _lease = ConcurrentTaskProbeLease {
probe: self,
task_id: task_id.clone(),
};
let fail = std::future::poll_fn(|task_context| {
let mut state = self.state.lock().unwrap();
*state.polls.entry(task_id.clone()).or_default() += 1;
state
.wakers
.insert(task_id.clone(), task_context.waker().clone());
state
.actions
.get(&task_id)
.copied()
.map_or(std::task::Poll::Pending, std::task::Poll::Ready)
})
.await;
if fail {
return Err(McpError::internal_error("concurrent supervisor failed"));
}
let result = serde_json::from_value(serde_json::json!({"content": []}))
.expect("bounded terminal result");
match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => {
initial.complete_task(result, None)?;
}
FinalTaskSupervisorHandoff::Resumed(accepted) => {
accepted.complete_task(result, None)?;
}
}
Ok(())
})
}
}
struct ConcurrentTaskFixture {
runtime: FinalTaskRuntime,
store: Arc<InMemoryFinalTaskStore>,
runner: AuthorizedTaskServiceRunner,
probe: Arc<ConcurrentTaskProbe>,
task_ids: Vec<FinalTaskId>,
}
fn concurrent_task_fixture(concurrency: usize, tasks: usize) -> ConcurrentTaskFixture {
let store = Arc::new(InMemoryFinalTaskStore::new(tasks).unwrap());
let task_ids = (0..tasks)
.map(|index| {
let task =
final_working_task_with_ttl(&format!("concurrent-task-{index:04}"), 60_000);
let task_id = task.base().task_id.clone();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.unwrap();
task_id
})
.collect();
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let probe = Arc::new(ConcurrentTaskProbe::default());
let supervisor: Arc<dyn ApplicationTaskSupervisor> = probe.clone();
let runner = runtime
.install_task_service(1, supervisor)
.unwrap()
.with_max_concurrent_handoffs(concurrency)
.unwrap();
ConcurrentTaskFixture {
runtime,
store,
runner,
probe,
task_ids,
}
}
fn check_concurrent_task_start(concurrency: usize) {
let mut fixture = concurrent_task_fixture(concurrency, 2);
let before = fixture.store.get_task(&fixture.task_ids[1]).unwrap();
let cx = Cx::for_testing();
let mut service = Box::pin(fixture.runner.run_service(&cx));
let mut task_context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(service.as_mut().poll(&mut task_context).is_pending());
assert!(fixture.runtime.is_task_service_ready());
let state = fixture.probe.state.lock().unwrap();
assert_eq!(state.started.len(), concurrency);
assert_eq!(state.live.len(), concurrency);
assert_eq!(state.peak_live, concurrency);
assert_eq!(
serde_json::to_value(fixture.store.get_task(&fixture.task_ids[1]).unwrap()).unwrap(),
serde_json::to_value(before).unwrap(),
"starting or queuing a sibling does not invent a public task transition"
);
if concurrency == 1 {
assert!(
!fixture
.store
.state
.lock()
.unwrap()
.handoff_leases
.contains_key(&fixture.task_ids[1])
);
}
drop(state);
drop(service);
assert!(!fixture.runtime.is_task_service_ready());
assert!(fixture.probe.state.lock().unwrap().live.is_empty());
assert!(
fixture
.store
.state
.lock()
.unwrap()
.handoff_leases
.is_empty()
);
}
#[test]
fn task_service_bounded_concurrency_starts_sibling_while_first_is_pending() {
check_concurrent_task_start(2);
}
#[test]
fn task_service_single_worker_keeps_sibling_queued() {
check_concurrent_task_start(1);
}
#[test]
fn task_service_concurrency_never_exceeds_limit_and_refills_released_slot() {
let mut fixture = concurrent_task_fixture(2, 3);
let cx = Cx::for_testing();
let mut service = Box::pin(fixture.runner.run_service(&cx));
let mut task_context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(service.as_mut().poll(&mut task_context).is_pending());
assert_eq!(fixture.probe.state.lock().unwrap().started.len(), 2);
assert!(
!fixture
.store
.state
.lock()
.unwrap()
.handoff_leases
.contains_key(&fixture.task_ids[2])
);
fixture.probe.release(&fixture.task_ids[0], false);
assert!(service.as_mut().poll(&mut task_context).is_pending());
{
let state = fixture.probe.state.lock().unwrap();
assert_eq!(state.started, fixture.task_ids);
assert_eq!(state.live.len(), 2);
assert_eq!(state.peak_live, 2);
}
assert!(matches!(
fixture.store.get_task(&fixture.task_ids[0]).unwrap(),
Some(FinalTask::Completed { .. })
));
for task_id in &fixture.task_ids[1..] {
fixture.probe.release(task_id, false);
}
assert!(service.as_mut().poll(&mut task_context).is_pending());
assert!(fixture.probe.state.lock().unwrap().live.is_empty());
for task_id in &fixture.task_ids {
assert!(matches!(
fixture.store.get_task(task_id).unwrap(),
Some(FinalTask::Completed { .. })
));
}
drop(service);
assert!(!fixture.runtime.is_task_service_ready());
}
#[test]
fn task_service_parallel_cancellation_leaves_sibling_running() {
let mut fixture = concurrent_task_fixture(2, 2);
let cx = Cx::for_testing();
let mut service = Box::pin(fixture.runner.run_service(&cx));
let mut task_context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(service.as_mut().poll(&mut task_context).is_pending());
let sibling_before = fixture.store.get_task(&fixture.task_ids[1]).unwrap();
let sibling_polls = fixture.probe.state.lock().unwrap().polls[&fixture.task_ids[1]];
fixture.runtime.cancel_task(&fixture.task_ids[0]).unwrap();
for _ in 0..3 {
assert!(service.as_mut().poll(&mut task_context).is_pending());
}
assert!(matches!(
fixture.store.get_task(&fixture.task_ids[0]).unwrap(),
Some(FinalTask::Cancelled(_))
));
{
let state = fixture.probe.state.lock().unwrap();
assert_eq!(state.live.len(), 1);
assert!(state.live.contains(&fixture.task_ids[1]));
assert_eq!(
state.polls[&fixture.task_ids[1]], sibling_polls,
"a task-local cancellation does not repoll its sibling"
);
}
assert_eq!(
serde_json::to_value(fixture.store.get_task(&fixture.task_ids[1]).unwrap()).unwrap(),
serde_json::to_value(sibling_before).unwrap()
);
assert!(fixture.runtime.is_task_service_ready());
fixture.probe.release(&fixture.task_ids[1], false);
assert!(service.as_mut().poll(&mut task_context).is_pending());
assert!(matches!(
fixture.store.get_task(&fixture.task_ids[1]).unwrap(),
Some(FinalTask::Completed { .. })
));
assert!(fixture.probe.state.lock().unwrap().live.is_empty());
drop(service);
assert!(!fixture.runtime.is_task_service_ready());
}
#[test]
fn task_service_parallel_drop_restores_all_claimed_handoffs() {
let mut fixture = concurrent_task_fixture(2, 3);
let cx = Cx::for_testing();
let before = fixture
.task_ids
.iter()
.map(|id| serde_json::to_value(fixture.store.get_task(id).unwrap()).unwrap())
.collect::<Vec<_>>();
let mut task_context = std::task::Context::from_waker(std::task::Waker::noop());
for invocation in 0..2 {
let mut service = Box::pin(fixture.runner.run_service(&cx));
assert!(service.as_mut().poll(&mut task_context).is_pending());
assert_eq!(fixture.probe.state.lock().unwrap().live.len(), 2);
drop(service);
assert!(!fixture.runtime.is_task_service_ready());
assert!(fixture.probe.state.lock().unwrap().live.is_empty());
let state = fixture.store.state.lock().unwrap();
assert!(state.handoff_leases.is_empty());
assert_eq!(
state.initial_work.len(),
3,
"all payloads survive invocation {invocation}"
);
drop(state);
assert!(
fixture
.runtime
.service_signal
.lock()
.unwrap()
.as_ref()
.unwrap()
.cancellation_wake
.state
.lock()
.unwrap()
.active
.is_empty()
);
let after = fixture
.task_ids
.iter()
.map(|id| serde_json::to_value(fixture.store.get_task(id).unwrap()).unwrap())
.collect::<Vec<_>>();
assert_eq!(
after, before,
"dropping the service preserves every public snapshot"
);
}
}
#[test]
fn task_service_parallel_supervisor_error_restores_siblings() {
for fail_before_first_poll in [false, true] {
let mut fixture = concurrent_task_fixture(2, 2);
let cx = Cx::for_testing();
let mut service = Box::pin(fixture.runner.run_service(&cx));
let mut task_context = std::task::Context::from_waker(std::task::Waker::noop());
if !fail_before_first_poll {
assert!(service.as_mut().poll(&mut task_context).is_pending());
assert_eq!(fixture.probe.state.lock().unwrap().live.len(), 2);
}
fixture.probe.release(&fixture.task_ids[0], true);
let std::task::Poll::Ready(Err(error)) = service.as_mut().poll(&mut task_context)
else {
panic!("a supervisor error terminates the bounded service");
};
assert_eq!(error.message, "concurrent supervisor failed");
assert!(!fixture.runtime.is_task_service_ready());
assert!(fixture.probe.state.lock().unwrap().live.is_empty());
let state = fixture.store.state.lock().unwrap();
assert!(state.handoff_leases.is_empty());
assert_eq!(
state.initial_work.len(),
2,
"even a claimed sibling that was never polled retains its payload"
);
assert!(
state
.tasks
.values()
.all(|task| matches!(task, FinalTask::Working(_)))
);
drop(state);
assert!(
fixture
.runtime
.service_signal
.lock()
.unwrap()
.as_ref()
.unwrap()
.cancellation_wake
.state
.lock()
.unwrap()
.active
.is_empty()
);
}
}
#[test]
fn task_service_parallel_retryable_low_id_failure_does_not_starve_sibling_on_restart() {
let mut fixture = concurrent_task_fixture(2, 2);
fixture.probe.release(&fixture.task_ids[0], true);
fixture.probe.release(&fixture.task_ids[1], false);
let cx = Cx::for_testing();
let mut task_context = std::task::Context::from_waker(std::task::Waker::noop());
for _ in 0..2 {
let mut service = Box::pin(fixture.runner.run_service(&cx));
assert!(matches!(
service.as_mut().poll(&mut task_context),
std::task::Poll::Ready(Err(_))
));
}
assert!(matches!(
fixture.store.get_task(&fixture.task_ids[0]).unwrap(),
Some(FinalTask::Working(_))
));
assert!(
matches!(
fixture.store.get_task(&fixture.task_ids[1]).unwrap(),
Some(FinalTask::Completed { .. })
),
"the first failed admission batch cannot pin recovery before the same failing task forever"
);
assert_eq!(
fixture.probe.state.lock().unwrap().started,
vec![
fixture.task_ids[0].clone(),
fixture.task_ids[1].clone(),
fixture.task_ids[0].clone()
]
);
assert!(!fixture.runtime.is_task_service_ready());
}
#[test]
fn task_service_parallel_mixed_recovery_kinds_do_not_starve_unpolled_sibling() {
let mut fixture = concurrent_task_fixture(2, 3);
let inputs: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}})).unwrap();
for task_id in &fixture.task_ids[1..] {
fixture
.runtime
.require_input(task_id, final_roots_request(), None)
.unwrap();
fixture.runtime.update_task(task_id, &inputs).unwrap();
}
fixture.probe.release(&fixture.task_ids[0], true);
fixture.probe.release(&fixture.task_ids[1], false);
fixture.probe.release(&fixture.task_ids[2], true);
let cx = Cx::for_testing();
let mut task_context = std::task::Context::from_waker(std::task::Waker::noop());
for _ in 0..2 {
let mut service = Box::pin(fixture.runner.run_service(&cx));
assert!(matches!(
service.as_mut().poll(&mut task_context),
std::task::Poll::Ready(Err(_))
));
}
assert!(matches!(
fixture.store.get_task(&fixture.task_ids[1]).unwrap(),
Some(FinalTask::Completed { .. })
));
for task_id in [&fixture.task_ids[0], &fixture.task_ids[2]] {
assert!(matches!(
fixture.store.get_task(task_id).unwrap(),
Some(FinalTask::Working(_))
));
}
assert_eq!(
fixture.probe.state.lock().unwrap().started,
vec![
fixture.task_ids[0].clone(),
fixture.task_ids[1].clone(),
fixture.task_ids[0].clone(),
]
);
assert!(!fixture.runtime.is_task_service_ready());
assert!(
fixture
.store
.state
.lock()
.unwrap()
.handoff_leases
.is_empty()
);
}
#[derive(Default)]
struct TaskTransitionCleanupProbe {
cleanup_ready: AtomicBool,
cleanup_waker: Mutex<Option<std::task::Waker>>,
resumed: AtomicUsize,
}
impl ApplicationTaskSupervisor for TaskTransitionCleanupProbe {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => {
if initial.work_descriptor().as_value().get("finish")
== Some(&serde_json::Value::Bool(true))
{
initial.complete_task(
serde_json::from_value(serde_json::json!({"content": []})).unwrap(),
None,
)?;
return Ok(());
}
initial.require_input(final_roots_request(), None)?;
std::future::poll_fn(|task_context| {
*self.cleanup_waker.lock().unwrap() =
Some(task_context.waker().clone());
if self.cleanup_ready.load(AtomicOrdering::SeqCst) {
std::task::Poll::Ready(())
} else {
std::task::Poll::Pending
}
})
.await;
}
FinalTaskSupervisorHandoff::Resumed(accepted) => {
assert!(
self.cleanup_ready.load(AtomicOrdering::SeqCst),
"successor application work cannot overlap predecessor cleanup"
);
self.resumed.fetch_add(1, AtomicOrdering::SeqCst);
let result =
serde_json::from_value(serde_json::json!({"content": []})).unwrap();
accepted.complete_task(result, None)?;
}
}
Ok(())
})
}
}
#[test]
fn task_service_parallel_successor_waits_for_predecessor_cleanup() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let task = final_working_task_with_ttl("concurrent-transition", 60_000);
let task_id = task.base().task_id.clone();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.unwrap();
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let probe = Arc::new(TaskTransitionCleanupProbe::default());
let supervisor: Arc<dyn ApplicationTaskSupervisor> = probe.clone();
let mut runner = runtime
.install_task_service(1, supervisor)
.unwrap()
.with_max_concurrent_handoffs(2)
.unwrap();
let cx = Cx::for_testing();
let mut service = Box::pin(runner.run_service(&cx));
let mut task_context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(service.as_mut().poll(&mut task_context).is_pending());
assert!(matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::InputRequired { .. })
));
let inputs = serde_json::from_value(serde_json::json!({"roots": {"roots": []}})).unwrap();
runtime.update_task(&task_id, &inputs).unwrap();
let after_update = serde_json::to_value(store.get_task(&task_id).unwrap()).unwrap();
assert!(
service.as_mut().poll(&mut task_context).is_pending(),
"a successor generation must not fail duplicate cancellation registration"
);
assert_eq!(probe.resumed.load(AtomicOrdering::SeqCst), 0);
assert_eq!(
serde_json::to_value(store.get_task(&task_id).unwrap()).unwrap(),
after_update
);
assert!(
test_next_accepted_input(&store).unwrap().is_some(),
"deferred successor input is restored for authoritative recovery"
);
probe.cleanup_ready.store(true, AtomicOrdering::SeqCst);
probe.cleanup_waker.lock().unwrap().take().unwrap().wake();
assert!(service.as_mut().poll(&mut task_context).is_pending());
assert_eq!(probe.resumed.load(AtomicOrdering::SeqCst), 1);
assert!(matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::Completed { .. })
));
assert!(runtime.is_task_service_ready());
drop(service);
assert!(!runtime.is_task_service_ready());
}
#[test]
fn task_02_parallel_cleanup_deferral_does_not_spend_successor_attempts() {
let (store, _clock) = execution_test_store(10_000, 5_000, 2);
let runtime = final_task_runtime(store.clone(), Arc::new(AtomicBool::new(false)));
let probe = Arc::new(TaskTransitionCleanupProbe::default());
let mut runner = runtime
.install_task_service(2, probe.clone())
.unwrap()
.with_max_concurrent_handoffs(2)
.unwrap();
RuntimeBuilder::current_thread()
.build()
.unwrap()
.block_on(async {
let cx = Cx::current().unwrap();
let mut service = Box::pin(runner.run_service(&cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(service.as_mut().poll(&mut context).is_pending());
let task = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.unwrap()
.task;
let task_id = task.base().task_id.clone();
assert!(service.as_mut().poll(&mut context).is_pending());
let inputs =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}})).unwrap();
runtime.update_task(&task_id, &inputs).unwrap();
let successor = store.get_task_snapshot(&task_id).unwrap().unwrap();
let budget = store
.task_execution_budget_if_current(&successor)
.unwrap()
.unwrap();
assert_eq!(budget.attempts(), 1);
assert!(budget.attempt_deadline().is_none());
for _ in 0..12 {
runtime.signal_task_service(task_id.clone());
assert!(service.as_mut().poll(&mut context).is_pending());
assert_eq!(
store.task_execution_budget_if_current(&successor).unwrap(),
Some(budget)
);
assert_eq!(probe.resumed.load(AtomicOrdering::SeqCst), 0);
}
let healthy = runtime
.create_task_with_work(
FinalTaskWorkDescriptor::new(serde_json::json!({"finish": true})).unwrap(),
None,
)
.unwrap()
.task;
assert!(service.as_mut().poll(&mut context).is_pending());
assert!(
matches!(
store.get_task(&healthy.base().task_id).unwrap(),
Some(FinalTask::Completed { .. })
),
"skipping a cleanup-owned successor must not hide healthy siblings"
);
assert_eq!(
store.task_execution_budget_if_current(&successor).unwrap(),
Some(budget)
);
probe.cleanup_ready.store(true, AtomicOrdering::SeqCst);
probe.cleanup_waker.lock().unwrap().take().unwrap().wake();
assert!(service.as_mut().poll(&mut context).is_pending());
assert_eq!(probe.resumed.load(AtomicOrdering::SeqCst), 1);
let completed = store.get_task_snapshot(&task_id).unwrap().unwrap();
assert!(matches!(completed.task(), FinalTask::Completed { .. }));
let final_budget = store
.task_execution_budget_if_current(&completed)
.unwrap()
.unwrap();
assert_eq!(final_budget.attempts(), 2);
assert_eq!(final_budget.total_deadline(), budget.total_deadline());
assert!(runtime.is_task_service_ready());
drop(service);
});
}
#[test]
fn task_service_concurrency_rejects_zero_and_over_limit() {
for limit in [1, MAX_CONCURRENT_TASK_HANDOFFS] {
let fixture = concurrent_task_fixture(limit, 1);
assert_eq!(fixture.runner.max_concurrent_handoffs(), limit);
}
for limit in [0, MAX_CONCURRENT_TASK_HANDOFFS + 1, usize::MAX] {
let fixture = concurrent_task_fixture(1, 1);
assert!(fixture.runner.with_max_concurrent_handoffs(limit).is_err());
assert!(!fixture.runtime.has_installed_task_service());
assert!(fixture.probe.state.lock().unwrap().started.is_empty());
assert_eq!(fixture.store.state.lock().unwrap().initial_work.len(), 1);
}
}
struct RecordingFinalTaskSupervisor {
accepted: Arc<Mutex<Vec<(FinalTaskId, FinalTaskInputResponses)>>>,
}
impl ApplicationTaskSupervisor for RecordingFinalTaskSupervisor {
fn resume<'a>(
&'a self,
cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let recorded = Arc::clone(&self.accepted);
Box::pin(async move {
let FinalTaskSupervisorHandoff::Resumed(accepted) = handoff else {
return Err(McpError::internal_error(
"recording supervisor expected a resumed task handoff",
));
};
recorded
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push((
accepted.task_id().clone(),
accepted.input_responses().clone(),
));
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
accepted.complete_task(result, None)?;
cx.cancel_with(CancelKind::User, None);
Ok(())
})
}
}
struct RecordingInitialFinalTaskSupervisor {
started: Arc<Mutex<Vec<(FinalTaskId, FinalTaskWorkDescriptor)>>>,
}
impl ApplicationTaskSupervisor for RecordingInitialFinalTaskSupervisor {
fn resume<'a>(
&'a self,
cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let started = Arc::clone(&self.started);
Box::pin(async move {
let FinalTaskSupervisorHandoff::Initial(initial) = handoff else {
return Err(McpError::internal_error(
"initial supervisor received a resumed task handoff",
));
};
started
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push((initial.task_id().clone(), initial.work_descriptor().clone()));
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
initial.complete_task(result, None)?;
cx.cancel_with(CancelKind::User, None);
Ok(())
})
}
}
struct CancellingAfterInitialHandoffsFinalTaskSupervisor {
started: Arc<AtomicUsize>,
cancel_after: usize,
}
struct RecordingQueuedWakeupSupervisor {
started: Arc<Mutex<Vec<FinalTaskId>>>,
}
impl ApplicationTaskSupervisor for RecordingQueuedWakeupSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
let FinalTaskSupervisorHandoff::Initial(initial) = handoff else {
return Err(McpError::internal_error(
"queued wakeup supervisor expected initial work",
));
};
self.started
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(initial.task_id().clone());
let result = serde_json::from_value(serde_json::json!({
"content": [{
"type": "text",
"text": initial.work_descriptor().as_value()["label"]
}]
}))
.expect("the queued task label forms a typed text result");
initial.complete_task(result, None)?;
Ok(())
})
}
}
struct RecordingRecoveryOrderFinalTaskSupervisor {
order: Arc<Mutex<Vec<&'static str>>>,
cancel_after: usize,
}
impl ApplicationTaskSupervisor for RecordingRecoveryOrderFinalTaskSupervisor {
fn resume<'a>(
&'a self,
cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let order = Arc::clone(&self.order);
let cancel_after = self.cancel_after;
Box::pin(async move {
let kind = match &handoff {
FinalTaskSupervisorHandoff::Initial(_) => "initial",
FinalTaskSupervisorHandoff::Resumed(_) => "resumed",
};
let observed = {
let mut order = order
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
order.push(kind);
order.len()
};
complete_final_task_handoff(handoff)?;
if observed == cancel_after {
cx.cancel_with(CancelKind::User, None);
}
Ok(())
})
}
}
impl ApplicationTaskSupervisor for CancellingAfterInitialHandoffsFinalTaskSupervisor {
fn resume<'a>(
&'a self,
cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let started = Arc::clone(&self.started);
let cancel_after = self.cancel_after;
Box::pin(async move {
let FinalTaskSupervisorHandoff::Initial(initial) = handoff else {
return Err(McpError::internal_error(
"bounded recovery supervisor expected an initial task handoff",
));
};
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
initial.complete_task(result, None)?;
if started.fetch_add(1, AtomicOrdering::SeqCst) + 1 == cancel_after {
cx.cancel_with(CancelKind::User, None);
}
Ok(())
})
}
}
struct FailingFinalTaskSupervisor;
impl ApplicationTaskSupervisor for FailingFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
_handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async {
Err(McpError::internal_error(
"planted caller-owned supervisor failure",
))
})
}
}
const RUN_SERVICE_SUPERVISOR_FAIL: usize = 0;
const RUN_SERVICE_SUPERVISOR_PENDING: usize = 1;
const RUN_SERVICE_SUPERVISOR_COMPLETE: usize = 2;
struct SwitchableRunServiceSupervisor {
action: Arc<AtomicUsize>,
}
impl ApplicationTaskSupervisor for SwitchableRunServiceSupervisor {
fn resume<'a>(
&'a self,
cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let action = Arc::clone(&self.action);
Box::pin(async move {
match action.load(AtomicOrdering::SeqCst) {
RUN_SERVICE_SUPERVISOR_FAIL => {
let _handoff = handoff;
Err(McpError::internal_error(
"planted retained-run-service supervisor failure",
))
}
RUN_SERVICE_SUPERVISOR_PENDING => {
let _handoff = handoff;
std::future::pending::<McpResult<()>>().await
}
RUN_SERVICE_SUPERVISOR_COMPLETE => {
complete_final_task_handoff(handoff)?;
cx.cancel_with(CancelKind::User, None);
Ok(())
}
_ => Err(McpError::internal_error(
"invalid retained-run-service test supervisor action",
)),
}
})
}
}
struct FailLowIdCompleteLaterInitialSupervisor {
low_task_id: FinalTaskId,
attempted: Arc<Mutex<Vec<FinalTaskId>>>,
}
impl ApplicationTaskSupervisor for FailLowIdCompleteLaterInitialSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let low_task_id = self.low_task_id.clone();
let attempted = Arc::clone(&self.attempted);
Box::pin(async move {
let FinalTaskSupervisorHandoff::Initial(initial) = handoff else {
return Err(McpError::internal_error(
"low-ID recovery fixture expected initial work",
));
};
let task_id = initial.task_id().clone();
attempted
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(task_id.clone());
if task_id == low_task_id {
return Err(McpError::internal_error(
"planted retryable low-ID recovery failure",
));
}
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
initial.complete_task(result, None)?;
Ok(())
})
}
}
struct CancelThenFailingFinalTaskSupervisor {
runtime: FinalTaskRuntime,
}
impl ApplicationTaskSupervisor for CancelThenFailingFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let runtime = self.runtime.clone();
Box::pin(async move {
runtime
.cancel_task(final_task_handoff_task_id(&handoff))
.expect("the elected task remains cancellable before the planted error");
Err(McpError::internal_error(
"planted supervisor error after cancellation election",
))
})
}
}
struct CancelThenHonoringCancellationFinalTaskSupervisor {
runtime: FinalTaskRuntime,
observed_cancellation: Arc<AtomicBool>,
}
impl ApplicationTaskSupervisor for CancelThenHonoringCancellationFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let runtime = self.runtime.clone();
let observed_cancellation = Arc::clone(&self.observed_cancellation);
Box::pin(async move {
runtime
.cancel_task(final_task_handoff_task_id(&handoff))
.expect("the elected task remains cancellable before honouring cancellation");
let cancellation_requested = match &handoff {
FinalTaskSupervisorHandoff::Initial(initial) => {
initial.is_cancellation_requested()
}
FinalTaskSupervisorHandoff::Resumed(accepted) => {
accepted.is_cancellation_requested()
}
}?;
observed_cancellation.store(cancellation_requested, AtomicOrdering::SeqCst);
if !cancellation_requested {
return Err(McpError::internal_error(
"the elected handoff did not observe its cancellation winner",
));
}
match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => initial.honor_cancellation(
Some("cancellation won the elected handoff".to_owned()),
),
FinalTaskSupervisorHandoff::Resumed(accepted) => accepted.honor_cancellation(
Some("cancellation won the elected handoff".to_owned()),
),
}?;
Ok(())
})
}
}
struct PendingFinalTaskSupervisor;
impl ApplicationTaskSupervisor for PendingFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
let _handoff = handoff;
std::future::pending::<McpResult<()>>().await
})
}
}
struct SignallingPendingFinalTaskSupervisor {
started: Sender<()>,
}
impl ApplicationTaskSupervisor for SignallingPendingFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let started = self.started.clone();
Box::pin(async move {
started.try_send(()).map_err(|_| {
McpError::internal_error(
"live pending-supervisor test lost its start notification",
)
})?;
let _handoff = handoff;
std::future::pending::<McpResult<()>>().await
})
}
}
struct PanickingFinalTaskSupervisor;
impl ApplicationTaskSupervisor for PanickingFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
let _handoff = handoff;
panic!("planted task supervisor panic");
})
}
}
struct ReadinessLeaseProbeFinalTaskStore {
inner: Arc<InMemoryFinalTaskStore>,
service_signal: Mutex<Option<Arc<Mutex<Option<FinalTaskServiceSignal>>>>>,
observed_ready_lease: AtomicBool,
}
impl ReadinessLeaseProbeFinalTaskStore {
fn new(inner: Arc<InMemoryFinalTaskStore>) -> Self {
Self {
inner,
service_signal: Mutex::new(None),
observed_ready_lease: AtomicBool::new(false),
}
}
fn observe_service_signal(
&self,
service_signal: Arc<Mutex<Option<FinalTaskServiceSignal>>>,
) {
*self
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(service_signal);
}
}
impl FinalTaskStore for ReadinessLeaseProbeFinalTaskStore {
delegate_final_execution_policy!();
fn create_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.create_task(task, notification)
}
fn create_task_with_work(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<FinalTaskSnapshot> {
let service_signal = self
.service_signal
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
.expect("test store receives the runtime service signal before public creation");
self.observed_ready_lease.store(
matches!(
service_signal.try_lock(),
Err(std::sync::TryLockError::WouldBlock)
),
AtomicOrdering::SeqCst,
);
self.inner
.create_task_with_work(task, notification, work_descriptor)
}
fn get_task(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTask>> {
self.inner.get_task(task_id)
}
fn get_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.get_task_snapshot(task_id)
}
fn replace_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.replace_task(task, notification)
}
fn replace_task_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<bool> {
self.inner
.replace_task_if_current(expected, task, notification)
}
fn request_cancellation(&self, task_id: &FinalTaskId) -> McpResult<()> {
self.inner.request_cancellation(task_id)
}
fn request_cancellation_if_current(&self, expected: &FinalTaskSnapshot) -> McpResult<bool> {
self.inner.request_cancellation_if_current(expected)
}
fn is_cancellation_requested(&self, task_id: &FinalTaskId) -> McpResult<bool> {
self.inner.is_cancellation_requested(task_id)
}
fn next_initial_work_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_initial_work_snapshot()
}
fn next_initial_work_snapshot_after(
&self,
after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_initial_work_snapshot_after(after_task_id)
}
fn take_initial_work_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
self.inner.take_initial_work_if_current(expected)
}
fn take_initial_work_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
self.inner
.take_initial_work_for_owner_if_current(expected, owner_id)
}
fn take_initial_work_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskInitialWorkClaim>> {
self.inner
.take_initial_work_handoff_for_owner_if_current(expected, owner_id)
}
fn restore_initial_work_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
self.inner
.restore_initial_work_if_current(task_id, generation, work_descriptor)
}
fn restore_initial_work_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: Option<u64>,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
self.inner.restore_initial_work_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
work_descriptor,
)
}
fn replace_task_and_append_input_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
input_responses: FinalTaskInputResponses,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.replace_task_and_append_input_if_current(
expected,
task,
notification,
input_responses,
)
}
fn replace_task_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner
.replace_task_and_clear_input_if_current(expected, task, notification)
}
fn take_input_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskInputResponses>> {
self.inner.take_input_if_current(expected)
}
fn take_input_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskInputResponses>> {
self.inner
.take_input_for_owner_if_current(expected, owner_id)
}
fn take_input_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskAcceptedInputClaim>> {
self.inner
.take_input_handoff_for_owner_if_current(expected, owner_id)
}
fn next_accepted_input_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_accepted_input_snapshot()
}
fn next_accepted_input_snapshot_after(
&self,
after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_accepted_input_snapshot_after(after_task_id)
}
fn restore_input_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
input_responses: FinalTaskInputResponses,
) -> McpResult<bool> {
self.inner
.restore_input_if_current(task_id, generation, input_responses)
}
fn restore_input_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: Option<u64>,
input_responses: FinalTaskInputResponses,
) -> McpResult<bool> {
self.inner.restore_input_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
input_responses,
)
}
fn begin_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<bool> {
self.inner
.begin_handoff_dispatch_if_current(task_id, generation)
}
fn begin_handoff_dispatch_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
) -> McpResult<Option<u64>> {
self.inner
.begin_handoff_dispatch_for_owner_if_current(task_id, generation, owner_id)
}
fn renew_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
self.inner.renew_handoff_dispatch_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
)
}
fn handoff_dispatch_lease_heartbeat_interval(&self) -> McpResult<StdDuration> {
self.inner.handoff_dispatch_lease_heartbeat_interval()
}
fn finish_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<bool> {
self.inner
.finish_handoff_dispatch_if_current(task_id, generation)
}
fn finish_handoff_dispatch_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
self.inner.finish_handoff_dispatch_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
)
}
fn request_cancellation_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
cancelled_task: FinalTask,
cancelled_notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.request_cancellation_and_clear_input_if_current(
expected,
cancelled_task,
cancelled_notification,
)
}
fn retention_clock_now(&self) -> Instant {
self.inner.retention_clock_now()
}
fn task_retention_deadline_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>> {
self.inner
.task_retention_deadline_if_current(task_id, generation)
}
}
struct LoseFirstAcceptedRecoveryCandidateStore {
inner: Arc<InMemoryFinalTaskStore>,
lose_first_take: Mutex<bool>,
}
impl LoseFirstAcceptedRecoveryCandidateStore {
fn new(inner: Arc<InMemoryFinalTaskStore>) -> Self {
Self {
inner,
lose_first_take: Mutex::new(true),
}
}
}
impl FinalTaskStore for LoseFirstAcceptedRecoveryCandidateStore {
delegate_final_execution_policy!();
fn create_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.create_task(task, notification)
}
fn get_task(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTask>> {
self.inner.get_task(task_id)
}
fn get_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.get_task_snapshot(task_id)
}
fn replace_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.replace_task(task, notification)
}
fn replace_task_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<bool> {
self.inner
.replace_task_if_current(expected, task, notification)
}
fn take_input_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskInputResponses>> {
let mut lose_first_take = self
.lose_first_take
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if *lose_first_take {
*lose_first_take = false;
if self
.inner
.take_input_for_owner_if_current(expected, owner_id)?
.is_some()
&& let Some(dispatch_fence) =
self.inner.begin_handoff_dispatch_for_owner_if_current(
&expected.task().base().task_id,
expected.generation(),
owner_id,
)?
{
let _ = self.inner.finish_handoff_dispatch_for_owner_if_current(
&expected.task().base().task_id,
expected.generation(),
owner_id,
dispatch_fence,
)?;
}
return Ok(None);
}
self.inner
.take_input_for_owner_if_current(expected, owner_id)
}
fn take_input_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskAcceptedInputClaim>> {
let mut lose_first_take = self
.lose_first_take
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if *lose_first_take {
*lose_first_take = false;
if self
.inner
.take_input_handoff_for_owner_if_current(expected, owner_id)?
.is_some()
&& let Some(dispatch_fence) =
self.inner.begin_handoff_dispatch_for_owner_if_current(
&expected.task().base().task_id,
expected.generation(),
owner_id,
)?
{
let _ = self.inner.finish_handoff_dispatch_for_owner_if_current(
&expected.task().base().task_id,
expected.generation(),
owner_id,
dispatch_fence,
)?;
}
return Ok(None);
}
self.inner
.take_input_handoff_for_owner_if_current(expected, owner_id)
}
fn next_accepted_input_snapshot_after(
&self,
after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_accepted_input_snapshot_after(after_task_id)
}
fn handoff_dispatch_lease_heartbeat_interval(&self) -> McpResult<StdDuration> {
self.inner.handoff_dispatch_lease_heartbeat_interval()
}
fn request_cancellation(&self, task_id: &FinalTaskId) -> McpResult<()> {
self.inner.request_cancellation(task_id)
}
fn request_cancellation_if_current(&self, expected: &FinalTaskSnapshot) -> McpResult<bool> {
self.inner.request_cancellation_if_current(expected)
}
fn is_cancellation_requested(&self, task_id: &FinalTaskId) -> McpResult<bool> {
self.inner.is_cancellation_requested(task_id)
}
fn retention_clock_now(&self) -> Instant {
self.inner.retention_clock_now()
}
fn task_retention_deadline_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>> {
self.inner
.task_retention_deadline_if_current(task_id, generation)
}
}
struct CancelBeforeFinalTaskDispatchStore {
inner: Arc<InMemoryFinalTaskStore>,
}
impl FinalTaskStore for CancelBeforeFinalTaskDispatchStore {
delegate_final_execution_policy!();
fn create_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.create_task(task, notification)
}
fn get_task(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTask>> {
self.inner.get_task(task_id)
}
fn get_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.get_task_snapshot(task_id)
}
fn replace_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.replace_task(task, notification)
}
fn replace_task_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<bool> {
self.inner
.replace_task_if_current(expected, task, notification)
}
fn restore_input_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: Option<u64>,
input_responses: FinalTaskInputResponses,
) -> McpResult<bool> {
self.inner.restore_input_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
input_responses,
)
}
fn take_input_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskAcceptedInputClaim>> {
self.inner
.take_input_handoff_for_owner_if_current(expected, owner_id)
}
fn begin_handoff_dispatch_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
) -> McpResult<Option<u64>> {
let Some(current) = self.inner.get_task_snapshot(task_id)? else {
return Ok(None);
};
let cancelled_task = FinalTask::Cancelled(transition_terminal_final_task_base(
current.task().base().clone(),
FinalTaskStatus::Cancelled,
None,
)?);
if current.generation() != generation
|| self
.inner
.request_cancellation_and_clear_input_if_current(
¤t,
cancelled_task.clone(),
final_task_notification(&cancelled_task),
)?
.is_none()
{
return Ok(None);
}
self.inner
.begin_handoff_dispatch_for_owner_if_current(task_id, generation, owner_id)
}
fn renew_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
self.inner.renew_handoff_dispatch_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
)
}
fn handoff_dispatch_lease_heartbeat_interval(&self) -> McpResult<StdDuration> {
self.inner.handoff_dispatch_lease_heartbeat_interval()
}
fn finish_handoff_dispatch_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
self.inner.finish_handoff_dispatch_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
)
}
fn request_cancellation(&self, task_id: &FinalTaskId) -> McpResult<()> {
self.inner.request_cancellation(task_id)
}
fn request_cancellation_if_current(&self, expected: &FinalTaskSnapshot) -> McpResult<bool> {
self.inner.request_cancellation_if_current(expected)
}
fn is_cancellation_requested(&self, task_id: &FinalTaskId) -> McpResult<bool> {
self.inner.is_cancellation_requested(task_id)
}
fn retention_clock_now(&self) -> Instant {
self.inner.retention_clock_now()
}
fn task_retention_deadline_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>> {
self.inner
.task_retention_deadline_if_current(task_id, generation)
}
}
struct AllowUnlimitedFinalTaskRetention;
impl FinalTaskRetentionAuthority for AllowUnlimitedFinalTaskRetention {
fn authorize_unlimited_retention(&self) -> McpResult<()> {
Ok(())
}
}
struct TerminalTransitionThenFailingFinalTaskSupervisor {}
impl ApplicationTaskSupervisor for TerminalTransitionThenFailingFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => initial
.complete_task(result, None)
.expect("the elected initial handoff commits the terminal task"),
FinalTaskSupervisorHandoff::Resumed(accepted) => accepted
.complete_task(result, None)
.expect("the elected resumed handoff commits the terminal task"),
};
Err(McpError::internal_error(
"planted supervisor failure after a newer transition",
))
})
}
}
struct FencedCompletingFinalTaskSupervisor;
impl ApplicationTaskSupervisor for FencedCompletingFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => initial
.complete_task(result, Some("completed by elected handoff".to_owned())),
FinalTaskSupervisorHandoff::Resumed(accepted) => accepted
.complete_task(result, Some("completed by elected handoff".to_owned())),
}?;
Ok(())
})
}
}
struct NoTransitionFinalTaskSupervisor;
impl ApplicationTaskSupervisor for NoTransitionFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
let _handoff = handoff;
Ok(())
})
}
}
struct StaleFenceCompletingFinalTaskSupervisor {
store: Arc<InMemoryFinalTaskStore>,
observed_error: Arc<Mutex<Option<McpError>>>,
}
impl ApplicationTaskSupervisor for StaleFenceCompletingFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let store = Arc::clone(&self.store);
let observed_error = Arc::clone(&self.observed_error);
Box::pin(async move {
let task_id = final_task_handoff_task_id(&handoff).clone();
{
let mut state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let lease = state
.handoff_leases
.get_mut(&task_id)
.expect("the elected handoff retains its exact in-memory dispatch lease");
let replacement_fence = lease
.dispatch_fence
.expect("the supervisor is invoked only after dispatch election")
.checked_add(1)
.expect("test dispatch fence remains representable");
lease.dispatch_fence = Some(replacement_fence);
state.next_dispatch_fence = replacement_fence;
}
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
let error = match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => {
initial.complete_task(result, Some("stale fence must fail".to_owned()))
}
FinalTaskSupervisorHandoff::Resumed(accepted) => {
accepted.complete_task(result, Some("stale fence must fail".to_owned()))
}
}
.expect_err("changing only the elected fence rejects the terminal transition");
*observed_error
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(error);
Ok(())
})
}
}
struct StaleGenerationCompletingFinalTaskSupervisor {
runtime: FinalTaskRuntime,
observed_error: Arc<Mutex<Option<McpError>>>,
}
impl ApplicationTaskSupervisor for StaleGenerationCompletingFinalTaskSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let runtime = self.runtime.clone();
let observed_error = Arc::clone(&self.observed_error);
Box::pin(async move {
let task_id = final_task_handoff_task_id(&handoff).clone();
runtime
.require_input(
&task_id,
final_roots_request(),
Some("newer generation won before completion".to_owned()),
)
.expect("the competing transition advances the durable generation");
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
let error = match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => {
initial.complete_task(result, Some("stale generation must fail".to_owned()))
}
FinalTaskSupervisorHandoff::Resumed(accepted) => accepted
.complete_task(result, Some("stale generation must fail".to_owned())),
}
.expect_err("a stale generation cannot commit a terminal handoff");
*observed_error
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(error);
Ok(())
})
}
}
struct RepeatedTerminalHandoffSupervisor {
observed_error: Arc<Mutex<Option<McpError>>>,
}
impl ApplicationTaskSupervisor for RepeatedTerminalHandoffSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let observed_error = Arc::clone(&self.observed_error);
Box::pin(async move {
let first_result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed first terminal task result");
match &handoff {
FinalTaskSupervisorHandoff::Initial(initial) => initial
.complete_task(first_result, Some("first terminal handoff".to_owned())),
FinalTaskSupervisorHandoff::Resumed(accepted) => accepted
.complete_task(first_result, Some("first terminal handoff".to_owned())),
}
.expect("the elected handoff commits its first terminal result");
let repeated_result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed repeated terminal task result");
let error =
match &handoff {
FinalTaskSupervisorHandoff::Initial(initial) => initial
.complete_task(repeated_result, Some("repeat must fail".to_owned())),
FinalTaskSupervisorHandoff::Resumed(accepted) => accepted
.complete_task(repeated_result, Some("repeat must fail".to_owned())),
}
.expect_err("a terminal handoff cannot commit twice");
*observed_error
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(error);
Ok(())
})
}
}
#[test]
fn auth_00_task_owner_retention_positive() {
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(store.clone(), Arc::new(AtomicBool::new(false)));
let owner = McpContext::new(Cx::for_testing(), 1)
.with_auth(fastmcp_core::AuthContext::with_subject("owner"));
let principal = task_request_principal(&owner).unwrap().unwrap();
let task = final_working_task_with_ttl("authenticated-retention", 60_000);
let task_id = task.base().task_id.clone();
store
.create_task_with_authenticated_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
principal,
)
.unwrap();
assert_eq!(
store
.next_initial_work_snapshot()
.unwrap()
.unwrap()
.authenticated_principal(),
Some(principal)
);
runtime
.require_input(&task_id, final_roots_request(), None)
.unwrap();
let current = runtime.task_for_request(&owner, &task_id).unwrap();
runtime
.update_task_snapshot(
¤t,
&serde_json::from_value(serde_json::json!({"roots": {"roots": []}})).unwrap(),
)
.unwrap();
assert_eq!(
store
.next_accepted_input_snapshot()
.unwrap()
.unwrap()
.authenticated_principal(),
Some(principal)
);
let current = runtime.task_for_request(&owner, &task_id).unwrap();
runtime.cancel_task_snapshot(¤t).unwrap();
assert!(runtime.notification_matches_task_owner(
¤t,
&store.latest_notification(&task_id).unwrap(),
));
assert_eq!(
runtime
.task_for_request(&owner, &task_id)
.unwrap()
.authenticated_principal(),
Some(principal)
);
*now.lock().unwrap() += StdDuration::from_millis(60_000);
assert!(store.get_task_snapshot(&task_id).unwrap().is_none());
assert!(
store
.state
.lock()
.unwrap()
.authenticated_principals
.is_empty()
);
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.unwrap();
assert!(
runtime.task_for_request(&owner, &task_id).is_err(),
"expired ownership cannot attach to a reused unprotected ID"
);
assert!(
!runtime.notification_matches_task_owner(
¤t,
&store.latest_notification(&task_id).unwrap(),
),
"an existing listener cannot inherit the replacement task"
);
assert!(
store
.get_task_snapshot(&task_id)
.unwrap()
.unwrap()
.authenticated_principal()
.is_none()
);
}
#[test]
fn auth_00_task_owner_retention_planted_negative() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(store.clone(), Arc::new(AtomicBool::new(false)));
let owner = McpContext::new(Cx::for_testing(), 1)
.with_auth(fastmcp_core::AuthContext::with_subject("owner"));
let foreign = McpContext::new(Cx::for_testing(), 1)
.with_auth(fastmcp_core::AuthContext::with_subject("foreign"));
let task = final_working_task_without_ttl("authenticated-denial");
let task_id = task.base().task_id.clone();
store
.create_task_with_authenticated_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
task_request_principal(&owner).unwrap().unwrap(),
)
.unwrap();
runtime
.require_input(&task_id, final_roots_request(), None)
.unwrap();
let before = store.get_task_snapshot(&task_id).unwrap().unwrap();
let before_notification =
serde_json::to_value(store.latest_notification(&task_id)).unwrap();
for caller in [&foreign, &McpContext::new(Cx::for_testing(), 1)] {
let error = runtime.task_for_request(caller, &task_id).unwrap_err();
assert_eq!(error.message, "Task not found");
assert_eq!(
store
.get_task_snapshot(&task_id)
.unwrap()
.unwrap()
.generation(),
before.generation()
);
assert_eq!(
serde_json::to_value(store.latest_notification(&task_id)).unwrap(),
before_notification
);
let state = store.state.lock().unwrap();
assert!(state.accepted_inputs.is_empty());
assert!(state.cancellation_requests.is_empty());
assert!(state.handoff_leases.is_empty());
assert_eq!(
state.authenticated_principals.get(&task_id).copied(),
before.authenticated_principal()
);
}
let current = runtime.task_for_request(&owner, &task_id).unwrap();
runtime.cancel_task_snapshot(¤t).unwrap();
assert!(
runtime.cancel_task_snapshot(&before).is_err(),
"stale authorized snapshot cannot overwrite a later generation"
);
let unsupported =
ReadinessLeaseProbeFinalTaskStore::new(Arc::new(InMemoryFinalTaskStore::default()));
assert!(
unsupported
.create_task_with_authenticated_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
task_request_principal(&owner).unwrap().unwrap(),
)
.is_err()
);
assert!(unsupported.get_task(&task_id).unwrap().is_none());
}
fn final_roots_request() -> FinalTaskInputRequests {
let mut requests = FinalTaskInputRequests::new();
requests.insert(
"roots".to_owned(),
serde_json::from_value(serde_json::json!({"method": "roots/list"}))
.expect("typed roots input request"),
);
requests
}
fn final_test_work_descriptor() -> FinalTaskWorkDescriptor {
FinalTaskWorkDescriptor::new(serde_json::json!({
"handler": "tasks-test",
"payload": {"fixture": "final-task"}
}))
.expect("non-null test work descriptor is valid")
}
fn complete_final_task_handoff(handoff: FinalTaskSupervisorHandoff) -> McpResult<()> {
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => {
let _ = initial.complete_task(result, None)?;
}
FinalTaskSupervisorHandoff::Resumed(accepted) => {
let _ = accepted.complete_task(result, None)?;
}
}
Ok(())
}
fn final_task_method_parameters(task_id: &FinalTaskId) -> serde_json::Value {
serde_json::json!({
"taskId": task_id,
"_meta": {
"io.modelcontextprotocol/protocolVersion": FINAL_PROTOCOL_VERSION,
"io.modelcontextprotocol/clientCapabilities": {}
}
})
}
fn enter_task_service_runner<'a>(
runner: AuthorizedTaskServiceRunner,
cx: &'a Cx,
) -> Pin<Box<dyn Future<Output = McpResult<()>> + 'a>> {
let mut running = Box::pin(runner.run(cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
Future::poll(running.as_mut(), &mut context),
std::task::Poll::Pending
));
running
}
fn poll_retained_task_service(
runner: &mut AuthorizedTaskServiceRunner,
cx: &Cx,
) -> std::task::Poll<McpResult<()>> {
let mut running = Box::pin(runner.run_service(cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
Future::poll(running.as_mut(), &mut context)
}
fn assert_exact_initial_work_is_recoverable(
store: &InMemoryFinalTaskStore,
task_id: &FinalTaskId,
work_descriptor: &FinalTaskWorkDescriptor,
) {
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(state.initial_work.get(task_id), Some(work_descriptor));
assert_eq!(state.work_descriptors.get(task_id), Some(work_descriptor));
assert!(
!state.handoff_leases.contains_key(task_id),
"a stopped retained service must not retain an initial-work lease"
);
assert!(
matches!(state.tasks.get(task_id), Some(FinalTask::Working(_))),
"a stopped retained service must leave the initial task working for retry"
);
}
fn assert_exact_accepted_input_is_recoverable(
store: &InMemoryFinalTaskStore,
task_id: &FinalTaskId,
input_responses: &FinalTaskInputResponses,
) {
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(state.accepted_inputs.get(task_id), Some(input_responses));
assert!(
!state.handoff_leases.contains_key(task_id),
"a stopped retained service must not retain an accepted-input lease"
);
assert!(
matches!(state.tasks.get(task_id), Some(FinalTask::Working(_))),
"a stopped retained service must leave the accepted input working for retry"
);
}
fn create_final_task_state_fixture(
runtime: &FinalTaskRuntime,
status_message: Option<String>,
) -> CreateTaskResult {
let task_id = generate_final_task_id().expect("generate final task ID for state fixture");
let now = final_task_timestamp().expect("generate final task timestamp for state fixture");
let task = FinalTask::Working(FinalTaskBase {
task_id,
status: FinalTaskStatus::Working,
status_message,
created_at: now.clone(),
last_updated_at: now,
ttl_ms: runtime
.config
.ttl_ms
.map(final_task_duration)
.transpose()
.expect("configured fixture TTL is a valid final task duration"),
poll_interval_ms: runtime
.config
.poll_interval_ms
.map(final_task_duration)
.transpose()
.expect("configured fixture poll interval is a valid final task duration"),
});
runtime
.persist_new_with_work(task.clone(), final_test_work_descriptor())
.expect("persist final task state fixture with durable work descriptor");
CreateTaskResult {
task,
meta: None,
additional: BTreeMap::new(),
}
}
fn create_accepted_final_input(
runtime: &FinalTaskRuntime,
input_responses: FinalTaskInputResponses,
) -> FinalTaskId {
let task_id = create_final_task_state_fixture(runtime, None)
.task
.base()
.task_id
.clone();
runtime
.require_input(&task_id, final_roots_request(), None)
.expect("task requests roots before accepted input");
runtime
.update_task(&task_id, &input_responses)
.expect("roots response returns the task to working");
task_id
}
fn final_working_task_without_ttl(task_id: &str) -> FinalTask {
let timestamp = FinalTaskTimestamp::parse("2026-07-28T12:00:00.000Z")
.expect("fixed test timestamp is valid");
FinalTask::Working(FinalTaskBase {
task_id: FinalTaskId::parse(task_id).expect("fixed test task ID is valid"),
status: FinalTaskStatus::Working,
status_message: None,
created_at: timestamp.clone(),
last_updated_at: timestamp,
ttl_ms: None,
poll_interval_ms: None,
})
}
fn final_working_task_with_ttl(task_id: &str, ttl_ms: u64) -> FinalTask {
let FinalTask::Working(mut base) = final_working_task_without_ttl(task_id) else {
unreachable!("the helper always constructs a working task");
};
base.ttl_ms = Some(final_task_duration(ttl_ms).expect("fixed test task TTL is valid"));
FinalTask::Working(base)
}
fn final_working_task_with_wire_durations(
task_id: &str,
ttl_ms: Option<&str>,
poll_interval_ms: Option<&str>,
) -> FinalTask {
let FinalTask::Working(mut base) = final_working_task_without_ttl(task_id) else {
unreachable!("the helper always constructs a working task");
};
base.ttl_ms = ttl_ms.map(|duration| {
serde_json::from_str(duration).expect("fixed test task TTL wire value is valid")
});
base.poll_interval_ms = poll_interval_ms.map(|duration| {
serde_json::from_str(duration)
.expect("fixed test task poll interval wire value is valid")
});
FinalTask::Working(base)
}
fn in_memory_store_with_test_clock(
max_tasks: usize,
) -> (Arc<InMemoryFinalTaskStore>, Arc<Mutex<Instant>>) {
let now = Arc::new(Mutex::new(Instant::now()));
let clock_now = Arc::clone(&now);
let clock: Arc<dyn Fn() -> Instant + Send + Sync> = Arc::new(move || {
*clock_now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
});
(
Arc::new(
InMemoryFinalTaskStore::with_clock(max_tasks, clock)
.expect("positive bounded store capacity is valid"),
),
now,
)
}
fn final_task_restoration_snapshot(
store: &InMemoryFinalTaskStore,
task_id: &FinalTaskId,
) -> (serde_json::Value, Option<Instant>, Option<Instant>) {
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let lease = state.handoff_leases.get(task_id);
(
serde_json::json!({
"task": state.tasks.get(task_id),
"principal": state.authenticated_principals.get(task_id).map(Sha256Digest::as_bytes),
"generation": state.generations.get(task_id),
"nextGeneration": state.next_generation,
"nextDispatchFence": state.next_dispatch_fence,
"payloadTotal": state.payload_accounting.total,
"payloadReserved": state.payload_accounting.reserved,
"payloadLimit": state.payload_accounting.limit,
"payloadCharges": state.payload_accounting.records,
"workDescriptor": state.work_descriptors.get(task_id).map(FinalTaskWorkDescriptor::as_value),
"initialWork": state.initial_work.get(task_id).map(FinalTaskWorkDescriptor::as_value),
"acceptedInputs": state.accepted_inputs.get(task_id),
"inputKeyHistory": state.input_key_history.get(task_id).map(|history| {
serde_json::json!({"keys": history.keys, "keyBytes": history.key_bytes})
}),
"notification": state.latest_notifications.get(task_id),
"cancellation": state.cancellation_requests.contains(task_id),
"lease": lease.map(|lease| serde_json::json!({
"generation": lease.generation,
"initial": lease.kind == InMemoryFinalTaskHandoffKind::Initial,
"elected": lease.dispatch_elected,
"owner": lease.owner_id,
"fence": lease.dispatch_fence,
})),
}),
state.expires_at.get(task_id).copied(),
lease.and_then(|lease| lease.recovery_expires_at),
)
}
fn encoded_final_task_test_bytes(value: &impl serde::Serialize) -> usize {
serde_json::to_vec(value)
.expect("test payload serializes")
.len()
}
fn assert_final_task_payload_accounting(store: &InMemoryFinalTaskStore) {
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut total = 0;
let mut reserved = 0;
for (task_id, task) in &state.tasks {
let mut actual = encoded_final_task_test_bytes(task)
+ encoded_final_task_test_bytes(
state
.latest_notifications
.get(task_id)
.expect("retained notification"),
);
for descriptor in [
state.work_descriptors.get(task_id),
state.initial_work.get(task_id),
]
.into_iter()
.flatten()
{
actual += encoded_final_task_test_bytes(descriptor.as_value());
}
if let Some(inputs) = state.accepted_inputs.get(task_id) {
actual += encoded_final_task_test_bytes(inputs);
}
if let Some(history) = state.input_key_history.get(task_id) {
actual += encoded_final_task_test_bytes(&history.keys);
}
assert_eq!(state.payload_accounting.records[task_id].total, actual);
let expected_reserve = if matches!(
task,
FinalTask::Working(_) | FinalTask::InputRequired { .. }
) {
FINAL_TASK_CANCELLATION_RESERVE_BYTES
} else {
0
};
assert_eq!(
state.payload_accounting.records[task_id].cancellation_reserve,
expected_reserve,
);
reserved += expected_reserve;
total += actual;
}
assert_eq!(state.payload_accounting.records.len(), state.tasks.len());
assert_eq!(state.payload_accounting.total, total);
assert_eq!(state.payload_accounting.reserved, reserved);
assert!(total + reserved <= state.payload_accounting.limit);
}
fn final_roots_input_with_encoded_bytes(key: &str, bytes: usize) -> FinalTaskInputResponses {
let mut wire = serde_json::json!({
(key): {"roots": [{"uri": "file:///task-input", "name": ""}]}
});
let overhead = encoded_final_task_test_bytes(&wire);
wire[key]["roots"][0]["name"] = serde_json::Value::String("x".repeat(bytes - overhead));
let inputs = serde_json::from_value(wire).expect("typed roots input fixture");
assert_eq!(encoded_final_task_test_bytes(&inputs), bytes);
inputs
}
fn final_update_precommit_fixture(
boundary: StdDuration,
) -> (Arc<InMemoryFinalTaskStore>, FinalTaskRuntime, FinalTaskId) {
let now = Arc::new(Mutex::new(Instant::now()));
let clock_now = Arc::clone(&now);
let clock_armed = Arc::new(AtomicBool::new(false));
let clock_sampled_boundary = Arc::new(AtomicBool::new(false));
let armed_for_clock = Arc::clone(&clock_armed);
let sampled_for_clock = Arc::clone(&clock_sampled_boundary);
let clock: Arc<dyn Fn() -> Instant + Send + Sync> = Arc::new(move || {
let sampled_now = *clock_now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if armed_for_clock.load(AtomicOrdering::SeqCst)
&& sampled_for_clock.swap(true, AtomicOrdering::SeqCst)
{
sampled_now
.checked_add(boundary)
.expect("precommit boundary remains representable")
} else {
sampled_now
}
});
let store = Arc::new(
InMemoryFinalTaskStore::with_clock(1, clock)
.expect("positive bounded store capacity is valid"),
);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(1_000, None)
.expect("finite update fixture retention is valid"),
Arc::new(|_| {}),
);
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
runtime
.require_input(&task_id, final_roots_request(), None)
.expect("update fixture enters input_required");
clock_armed.store(true, AtomicOrdering::SeqCst);
(store, runtime, task_id)
}
struct RuntimeBoundaryProbeFinalTaskStore {
snapshot: Mutex<FinalTaskSnapshot>,
execution_budget: Mutex<FinalTaskExecutionBudget>,
transition_write_calls: AtomicUsize,
transition_result_override: Mutex<Option<FinalTask>>,
work_descriptor: Mutex<Option<FinalTaskWorkDescriptor>>,
accepted_inputs: Mutex<Option<FinalTaskInputResponses>>,
initial_claim_override: Mutex<Option<FinalTaskInitialWorkClaim>>,
accepted_claim_override: Mutex<Option<FinalTaskAcceptedInputClaim>>,
cancellation_result_override: Mutex<Option<FinalTaskSnapshot>>,
force_false_cas: AtomicBool,
}
impl RuntimeBoundaryProbeFinalTaskStore {
fn new(task: FinalTask) -> Self {
Self {
snapshot: Mutex::new(FinalTaskSnapshot::new(task, 1)),
execution_budget: Mutex::new(
new_in_memory_final_task_execution_budget(
FinalTaskExecutionLimits::default(),
Instant::now(),
)
.expect("default probe deadline fits the process clock"),
),
transition_write_calls: AtomicUsize::new(0),
transition_result_override: Mutex::new(None),
work_descriptor: Mutex::new(Some(final_test_work_descriptor())),
accepted_inputs: Mutex::new(None),
initial_claim_override: Mutex::new(None),
accepted_claim_override: Mutex::new(None),
cancellation_result_override: Mutex::new(None),
force_false_cas: AtomicBool::new(false),
}
}
fn snapshot(&self) -> FinalTaskSnapshot {
self.snapshot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
fn replace_snapshot_for_read(&self, task: FinalTask) {
let mut snapshot = self
.snapshot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*snapshot = FinalTaskSnapshot::new(task, snapshot.generation());
}
fn set_accepted_inputs(&self, input_responses: FinalTaskInputResponses) {
*self
.accepted_inputs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(input_responses);
}
}
impl FinalTaskStore for RuntimeBoundaryProbeFinalTaskStore {
fn execution_limits(&self) -> McpResult<FinalTaskExecutionLimits> {
Ok(FinalTaskExecutionLimits::default())
}
fn task_execution_budget_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskExecutionBudget>> {
let snapshot = self.snapshot.lock().unwrap();
if snapshot.generation() != expected.generation()
|| snapshot.task().base().task_id != expected.task().base().task_id
{
return Ok(None);
}
Ok(Some(*self.execution_budget.lock().unwrap()))
}
fn expire_task_execution_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskSnapshot>> {
let mut snapshot = self.snapshot.lock().unwrap();
if snapshot.generation() != expected.generation()
|| snapshot.task().base().task_id != expected.task().base().task_id
|| !matches!(
snapshot.task(),
FinalTask::Working(_) | FinalTask::InputRequired { .. }
)
|| Instant::now() < self.execution_budget.lock().unwrap().total_deadline()
{
return Ok(None);
}
let task = FinalTask::Failed {
base: transition_terminal_final_task_base(
snapshot.task().base().clone(),
FinalTaskStatus::Failed,
None,
)?,
error: FinalTaskError {
code: (-32000).into(),
message: FINAL_TASK_TOTAL_DEADLINE_EXCEEDED.to_owned(),
data: None,
additional: BTreeMap::new(),
},
};
*snapshot = FinalTaskSnapshot::new(task, snapshot.generation() + 1);
Ok(Some(snapshot.clone()))
}
fn next_expired_execution_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
let snapshot = self.snapshot();
Ok((matches!(
snapshot.task(),
FinalTask::Working(_) | FinalTask::InputRequired { .. }
) && Instant::now() >= self.execution_budget.lock().unwrap().total_deadline())
.then_some(snapshot))
}
fn create_task(
&self,
task: FinalTask,
_notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.replace_snapshot_for_read(task);
Ok(())
}
fn create_task_with_work(
&self,
task: FinalTask,
_notification: FinalTaskStatusNotification,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<FinalTaskSnapshot> {
let mut snapshot = self.snapshot.lock().unwrap();
let budget = new_in_memory_final_task_execution_budget(
FinalTaskExecutionLimits::default(),
Instant::now(),
)?;
*snapshot = FinalTaskSnapshot::new(task, snapshot.generation());
*self.execution_budget.lock().unwrap() = budget;
*self
.work_descriptor
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(work_descriptor);
Ok(snapshot.clone())
}
fn get_task(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTask>> {
let snapshot = self.snapshot();
if &snapshot.task().base().task_id != task_id {
return Ok(None);
}
Ok(Some(snapshot.into_task()))
}
fn get_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTaskSnapshot>> {
let snapshot = self.snapshot();
if &snapshot.task().base().task_id != task_id {
return Ok(None);
}
Ok(Some(snapshot))
}
fn replace_task(
&self,
task: FinalTask,
_notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.replace_snapshot_for_read(task);
Ok(())
}
fn replace_task_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
_notification: FinalTaskStatusNotification,
) -> McpResult<bool> {
if self.force_false_cas.load(AtomicOrdering::SeqCst) {
return Ok(false);
}
let mut snapshot = self
.snapshot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if snapshot.generation() != expected.generation() {
return Ok(false);
}
let next_generation = snapshot
.generation()
.checked_add(1)
.ok_or_else(|| McpError::internal_error("probe generation exhausted"))?;
let committed_task = self
.transition_result_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
.unwrap_or(task);
*snapshot = FinalTaskSnapshot::new(committed_task, next_generation);
Ok(true)
}
fn replace_task_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
_notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.transition_write_calls
.fetch_add(1, AtomicOrdering::SeqCst);
if self.force_false_cas.load(AtomicOrdering::SeqCst) {
return Ok(None);
}
let mut snapshot = self.snapshot.lock().unwrap();
if snapshot.generation() != expected.generation() {
return Ok(None);
}
let generation = snapshot
.generation()
.checked_add(1)
.ok_or_else(|| McpError::internal_error("probe generation exhausted"))?;
let committed_task = self
.transition_result_override
.lock()
.unwrap()
.clone()
.unwrap_or(task);
*snapshot = FinalTaskSnapshot::new(committed_task, generation);
Ok(Some(snapshot.clone()))
}
fn next_initial_work_snapshot_after(
&self,
_after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
Ok(Some(self.snapshot()))
}
fn take_initial_work_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskInitialWorkClaim>> {
if let Some(claim) = self
.initial_claim_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
{
return Ok(Some(claim));
}
Ok(self
.work_descriptor
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
.map(|work_descriptor| {
FinalTaskInitialWorkClaim::new(
expected.task().base().task_id.clone(),
expected.generation(),
owner_id,
work_descriptor,
)
}))
}
fn next_accepted_input_snapshot_after(
&self,
_after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
Ok(self
.accepted_inputs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_some()
.then(|| self.snapshot()))
}
fn take_input_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskAcceptedInputClaim>> {
if let Some(claim) = self
.accepted_claim_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
{
return Ok(Some(claim));
}
let Some(input_responses) = self
.accepted_inputs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
else {
return Ok(None);
};
let work_descriptor = self
.work_descriptor
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
.ok_or_else(|| McpError::internal_error("probe descriptor missing"))?;
Ok(Some(FinalTaskAcceptedInputClaim::new(
expected.task().base().task_id.clone(),
expected.generation(),
owner_id,
work_descriptor,
input_responses,
)))
}
fn request_cancellation_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
cancelled_task: FinalTask,
_cancelled_notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
if let Some(snapshot) = self
.cancellation_result_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
{
return Ok(Some(snapshot));
}
let mut snapshot = self
.snapshot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.force_false_cas.load(AtomicOrdering::SeqCst)
|| snapshot.generation() != expected.generation()
{
return Ok(None);
}
let generation = snapshot
.generation()
.checked_add(1)
.ok_or_else(|| McpError::internal_error("probe generation exhausted"))?;
*snapshot = FinalTaskSnapshot::new(cancelled_task, generation);
Ok(Some(snapshot.clone()))
}
fn request_cancellation(&self, _task_id: &FinalTaskId) -> McpResult<()> {
Ok(())
}
fn request_cancellation_if_current(&self, expected: &FinalTaskSnapshot) -> McpResult<bool> {
Ok(!self.force_false_cas.load(AtomicOrdering::SeqCst)
&& self.snapshot().generation() == expected.generation())
}
fn is_cancellation_requested(&self, _task_id: &FinalTaskId) -> McpResult<bool> {
Ok(true)
}
fn retention_clock_now(&self) -> Instant {
Instant::now()
}
fn task_retention_deadline_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>> {
let snapshot = self.snapshot();
if &snapshot.task().base().task_id != task_id || snapshot.generation() != generation {
return Ok(None);
}
Ok(Some(FinalTaskRetentionDeadline::Unlimited))
}
}
#[test]
fn task_03_in_memory_runtime_constructor_lifecycle_positive() {
let runtime = FinalTaskRuntime::in_memory(
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid in-memory task policy"),
Arc::new(|_| {}),
);
let service_runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("installing a caller-owned service reserves the runner");
let service_cx = Cx::for_testing();
let _running_service = enter_task_service_runner(service_runner, &service_cx);
let task_id = runtime
.create_task_with_work(final_test_work_descriptor(), Some("accepted".to_owned()))
.expect("the shipped in-memory runtime creates a task")
.task
.base()
.task_id
.clone();
assert!(matches!(
runtime
.get_task(&task_id)
.expect("created task remains readable")
.task,
FinalTask::Working(_)
));
runtime
.cancel_task(&task_id)
.expect("created unelected task accepts terminal cancellation");
assert!(matches!(
runtime
.get_task(&task_id)
.expect("terminally cancelled task remains readable")
.task,
FinalTask::Cancelled(_)
));
assert!(
!runtime
.is_cancellation_requested(&task_id)
.expect("terminal task has no outstanding cooperative intent"),
"an unelected task is cancelled atomically instead of leaving worker intent"
);
}
#[test]
fn task_03_runtime_accepts_bound_custom_store_handoffs() {
let initial = final_working_task_without_ttl("task-runtime-custom-handoff-positive");
let task_id = initial.base().task_id.clone();
let store = Arc::new(RuntimeBoundaryProbeFinalTaskStore::new(initial));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| {}),
);
let initial_work = runtime
.recover_initial_work()
.expect("a bound initial-work claim is accepted")
.expect("the valid custom store provides initial work");
assert_eq!(initial_work.task_id(), &task_id);
assert_eq!(initial_work.generation(), store.snapshot().generation());
assert_eq!(
initial_work.work_descriptor(),
&final_test_work_descriptor()
);
let responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed accepted-input fixture");
store.set_accepted_inputs(responses.clone());
let accepted_input = runtime
.recover_accepted_input()
.expect("a bound accepted-input claim is accepted")
.expect("the valid custom store provides accepted input");
assert_eq!(accepted_input.task_id(), &task_id);
assert_eq!(accepted_input.generation(), store.snapshot().generation());
assert_eq!(accepted_input.input_responses(), &responses);
}
#[test]
fn task_03_runtime_accepts_valid_custom_store_create_with_work() {
let initial = final_working_task_without_ttl("task-runtime-custom-create-positive-old");
let store = Arc::new(RuntimeBoundaryProbeFinalTaskStore::new(initial));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| {}),
);
let created = final_working_task_without_ttl("task-runtime-custom-create-positive-new");
runtime
.persist_new_with_work(created.clone(), final_test_work_descriptor())
.expect("a valid custom-store create-with-work remains accepted");
let committed = store.snapshot();
assert!(
final_tasks_match_exactly(committed.task(), &created).expect("compare created task"),
"the valid custom store retains exactly the create-with-work task"
);
}
#[test]
fn task_03_runtime_rejects_malformed_custom_store_read_without_mutating_the_snapshot() {
let valid = final_working_task_without_ttl("task-runtime-custom-read");
let task_id = valid.base().task_id.clone();
let store = Arc::new(RuntimeBoundaryProbeFinalTaskStore::new(valid.clone()));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| {}),
);
assert!(matches!(
runtime
.get_task(&task_id)
.expect("the otherwise identical valid custom-store task is admitted")
.task,
FinalTask::Working(_)
));
let FinalTask::Working(mut malformed_base) = valid else {
unreachable!("the fixture is a working task");
};
malformed_base.status = FinalTaskStatus::Cancelled;
store.replace_snapshot_for_read(FinalTask::Working(malformed_base));
let before = store.snapshot();
let before_wire = serde_json::to_value(before.task())
.expect("serialize malformed custom-store snapshot before rejection");
let error = runtime
.get_task(&task_id)
.expect_err("changing only the retained status must reject the custom-store read");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
let after = store.snapshot();
assert_eq!(
serde_json::to_value(after.task())
.expect("serialize malformed custom-store snapshot after rejection"),
before_wire,
"the runtime must not rewrite a malformed custom-store read"
);
assert_eq!(
after.generation(),
before.generation(),
"a rejected custom-store read cannot advance its generation"
);
}
#[test]
fn task_03_runtime_write_boundary_rejects_malformed_custom_store_transition_unchanged() {
let initial = final_working_task_without_ttl("task-runtime-custom-write");
let task_id = initial.base().task_id.clone();
let store = Arc::new(RuntimeBoundaryProbeFinalTaskStore::new(initial));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| {}),
);
let initial_snapshot = runtime
.load_task_snapshot(&task_id)
.expect("the valid custom-store snapshot is admitted at the runtime boundary");
let FinalTask::Working(mut updated_base) = initial_snapshot.task().clone() else {
unreachable!("the fixture begins in the working state");
};
updated_base.status_message = Some("accepted update".to_owned());
let accepted = FinalTask::Working(updated_base);
runtime
.persist_transition_clearing_input(&initial_snapshot, accepted.clone())
.expect("the custom store receives an otherwise valid transition");
assert_eq!(
store.transition_write_calls.load(AtomicOrdering::SeqCst),
1,
"the accepted transition reaches the arbitrary store exactly once"
);
let before_rejection = store.snapshot();
let before_wire = serde_json::to_value(before_rejection.task())
.expect("serialize custom-store snapshot before malformed write");
let FinalTask::Working(mut malformed_base) = accepted else {
unreachable!("the accepted transition remains working");
};
malformed_base.status = FinalTaskStatus::Cancelled;
let malformed = FinalTask::Working(malformed_base);
let error = runtime
.persist_transition_clearing_input(&before_rejection, malformed)
.expect_err(
"changing only the status/variant alignment rejects before the store write",
);
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
assert_eq!(
store.transition_write_calls.load(AtomicOrdering::SeqCst),
1,
"the malformed transition cannot reach an arbitrary store implementation"
);
let after_rejection = store.snapshot();
assert_eq!(
serde_json::to_value(after_rejection.task())
.expect("serialize custom-store snapshot after malformed write rejection"),
before_wire,
"the near-identical rejected write leaves the durable task unchanged"
);
assert_eq!(
after_rejection.generation(),
before_rejection.generation(),
"the rejected write cannot advance the arbitrary store generation"
);
}
#[test]
fn task_03_runtime_rejects_custom_store_create_descriptor_and_retention_drift_unchanged() {
let initial = final_working_task_without_ttl("task-runtime-custom-create");
let task_id = initial.base().task_id.clone();
let store = Arc::new(RuntimeBoundaryProbeFinalTaskStore::new(initial));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| {}),
);
let before = store.snapshot();
let invalid_descriptor = FinalTaskWorkDescriptor(serde_json::Value::Null);
let error = runtime
.persist_new_with_work(
final_working_task_without_ttl("task-runtime-custom-new"),
invalid_descriptor,
)
.expect_err("a null work descriptor rejects before a permissive create");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
let FinalTask::Working(mut identity_drifted_base) = before.task().clone() else {
unreachable!("the probe begins working");
};
identity_drifted_base.task_id = FinalTaskId::parse("task-runtime-custom-create-other")
.expect("fixed replacement task ID is valid");
let error = runtime
.persist_transition_clearing_input(&before, FinalTask::Working(identity_drifted_base))
.expect_err("changing only task identity rejects before the store write");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
let FinalTask::Working(mut drifted_base) = before.task().clone() else {
unreachable!("the probe begins working");
};
drifted_base.ttl_ms = Some(serde_json::from_str("1").expect("valid ttl fixture"));
let error = runtime
.persist_transition_clearing_input(&before, FinalTask::Working(drifted_base))
.expect_err("changing only retained ttl rejects before the store write");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
let after = store.snapshot();
assert!(
final_tasks_match_exactly(before.task(), after.task()).expect("compare probe task"),
"create and retention rejections leave the permissive store unchanged"
);
assert_eq!(after.generation(), before.generation());
assert_eq!(after.task().base().task_id, task_id);
}
#[test]
fn task_03_runtime_rejects_custom_store_recovery_claim_binding_substitution_unchanged() {
let initial = final_working_task_without_ttl("task-runtime-custom-recovery");
let task_id = initial.base().task_id.clone();
let store = Arc::new(RuntimeBoundaryProbeFinalTaskStore::new(initial));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| {}),
);
let baseline = store.snapshot();
*store
.initial_claim_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(FinalTaskInitialWorkClaim::new(
task_id.clone(),
baseline.generation(),
"different-owner",
final_test_work_descriptor(),
));
let error = runtime
.recover_initial_work()
.expect_err("changing only the owner rejects an initial recovery handoff");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
*store
.initial_claim_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(FinalTaskInitialWorkClaim::new(
FinalTaskId::parse("task-runtime-custom-recovery-other")
.expect("fixed substituted task ID is valid"),
baseline.generation(),
FINAL_TASK_TEST_DIRECT_OWNER,
final_test_work_descriptor(),
));
let error = runtime
.recover_initial_work()
.expect_err("changing only task identity rejects an initial recovery handoff");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
*store
.initial_claim_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(FinalTaskInitialWorkClaim::new(
task_id.clone(),
baseline.generation(),
FINAL_TASK_TEST_DIRECT_OWNER,
FinalTaskWorkDescriptor(serde_json::Value::Null),
));
let error = runtime
.recover_initial_work()
.expect_err("changing only the work descriptor rejects before application handoff");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
let responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed accepted-input fixture");
store.set_accepted_inputs(responses.clone());
*store
.accepted_claim_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(FinalTaskAcceptedInputClaim::new(
task_id.clone(),
baseline.generation().saturating_add(1),
FINAL_TASK_TEST_DIRECT_OWNER,
final_test_work_descriptor(),
responses,
));
let error = runtime
.recover_accepted_input()
.expect_err("changing only the generation rejects an accepted-input handoff");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
*store
.accepted_claim_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(FinalTaskAcceptedInputClaim::new(
task_id,
baseline.generation(),
FINAL_TASK_TEST_DIRECT_OWNER,
final_test_work_descriptor(),
FinalTaskInputResponses::new(),
));
let error = runtime
.recover_accepted_input()
.expect_err("changing only accepted-input payload emptiness rejects before handoff");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
let after = store.snapshot();
assert!(
final_tasks_match_exactly(baseline.task(), after.task()).expect("compare probe task"),
"rejected recovery claims never rewrite the retained task"
);
assert_eq!(after.generation(), baseline.generation());
}
type CommittedSnapshotProbe = Box<dyn FnOnce(FinalTaskSnapshot) -> FinalTaskSnapshot + Send>;
struct CommittedSnapshotProbeStore {
inner: Arc<InMemoryFinalTaskStore>,
after_commit: Mutex<Option<CommittedSnapshotProbe>>,
fail_observation: AtomicBool,
}
impl CommittedSnapshotProbeStore {
fn committed(&self, snapshot: FinalTaskSnapshot) -> FinalTaskSnapshot {
let probe = self.after_commit.lock().unwrap().take();
match probe {
Some(probe) => probe(snapshot),
None => snapshot,
}
}
}
impl FinalTaskStore for CommittedSnapshotProbeStore {
delegate_final_execution_policy!();
fn create_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.create_task(task, notification)
}
fn create_task_with_work(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<FinalTaskSnapshot> {
self.inner
.create_task_with_work(task, notification, work_descriptor)
.map(|snapshot| self.committed(snapshot))
}
fn create_task_with_authenticated_work(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
work_descriptor: FinalTaskWorkDescriptor,
principal: Sha256Digest,
) -> McpResult<FinalTaskSnapshot> {
self.inner
.create_task_with_authenticated_work(task, notification, work_descriptor, principal)
.map(|snapshot| self.committed(snapshot))
}
fn get_task(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTask>> {
self.inner.get_task(task_id)
}
fn get_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTaskSnapshot>> {
if self.fail_observation.load(AtomicOrdering::SeqCst) {
return Err(McpError::internal_error(
"planted post-commit observation failure",
));
}
self.inner.get_task_snapshot(task_id)
}
fn replace_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.replace_task(task, notification)
}
fn replace_task_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<bool> {
self.inner
.replace_task_if_current(expected, task, notification)
}
fn replace_task_and_append_input_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
input_responses: FinalTaskInputResponses,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner
.replace_task_and_append_input_if_current(
expected,
task,
notification,
input_responses,
)
.map(|snapshot| snapshot.map(|snapshot| self.committed(snapshot)))
}
fn replace_task_and_clear_input_for_handoff_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
dispatch_fence: u64,
cancellation_required: bool,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner
.replace_task_and_clear_input_for_handoff_if_current(
expected,
owner_id,
dispatch_fence,
cancellation_required,
task,
notification,
)
.map(|snapshot| snapshot.map(|snapshot| self.committed(snapshot)))
}
fn next_initial_work_snapshot_after(
&self,
after: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_initial_work_snapshot_after(after)
}
fn next_accepted_input_snapshot_after(
&self,
after: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_accepted_input_snapshot_after(after)
}
fn request_cancellation(&self, task_id: &FinalTaskId) -> McpResult<()> {
self.inner.request_cancellation(task_id)
}
fn request_cancellation_if_current(&self, expected: &FinalTaskSnapshot) -> McpResult<bool> {
self.inner.request_cancellation_if_current(expected)
}
fn is_cancellation_requested(&self, task_id: &FinalTaskId) -> McpResult<bool> {
self.inner.is_cancellation_requested(task_id)
}
fn retention_clock_now(&self) -> Instant {
self.inner.retention_clock_now()
}
fn task_retention_deadline_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>> {
self.inner
.task_retention_deadline_if_current(task_id, generation)
}
}
#[derive(Clone, Copy, Debug)]
enum AtomicTaskCommitPhase {
Create,
Update,
RequireInput,
}
#[derive(Clone, Copy, Debug)]
enum AtomicTaskReceiptMutation {
Identity,
Payload,
Principal,
Generation,
OlderGeneration,
}
fn assert_atomic_task_receipt(
phase: AtomicTaskCommitPhase,
authenticated: bool,
mutation: Option<AtomicTaskReceiptMutation>,
) {
const OWNER: &str = "atomic-receipt-worker";
let inner = Arc::new(InMemoryFinalTaskStore::default());
let notifications = Arc::new(Mutex::new(Vec::new()));
let worker_notifications = notifications.clone();
let worker = FinalTaskRuntime::new(
inner.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).unwrap(),
Arc::new(move |notification| worker_notifications.lock().unwrap().push(notification)),
);
let probe = Arc::new(CommittedSnapshotProbeStore {
inner: inner.clone(),
after_commit: Mutex::new(None),
fail_observation: AtomicBool::new(false),
});
let request_notifications = notifications.clone();
let runtime = FinalTaskRuntime::new(
probe.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).unwrap(),
Arc::new(move |notification| request_notifications.lock().unwrap().push(notification)),
);
let mut ctx = McpContext::new(Cx::for_testing(), 1);
if authenticated {
ctx = ctx.with_auth(fastmcp_core::AuthContext::with_subject("atomic-owner"));
}
let principal = task_request_principal(&ctx).unwrap();
let retained_after_worker = Arc::new(Mutex::new(None));
let after_worker = retained_after_worker.clone();
let worker_inner = inner.clone();
let worker_for_probe = worker.clone();
let predecessor_generation = Arc::new(AtomicU64::new(0));
let generation_for_probe = predecessor_generation.clone();
let hook: CommittedSnapshotProbe = Box::new(move |mut committed| {
let task_id = committed.task().base().task_id.clone();
match phase {
AtomicTaskCommitPhase::Create | AtomicTaskCommitPhase::Update => {
let claimed = match phase {
AtomicTaskCommitPhase::Create => worker_inner
.take_initial_work_handoff_for_owner_if_current(&committed, OWNER)
.unwrap()
.is_some(),
AtomicTaskCommitPhase::Update => worker_inner
.take_input_handoff_for_owner_if_current(&committed, OWNER)
.unwrap()
.is_some(),
AtomicTaskCommitPhase::RequireInput => unreachable!(),
};
assert!(
claimed,
"the real store provides the exact committed handoff"
);
let fence = worker_inner
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
committed.generation(),
OWNER,
)
.unwrap()
.unwrap();
let successor = match phase {
AtomicTaskCommitPhase::Create => FinalTask::InputRequired {
base: transition_final_task_base(committed.task().base().clone(), FinalTaskStatus::InputRequired, None).unwrap(),
input_requests: final_roots_request(),
},
AtomicTaskCommitPhase::Update => FinalTask::Completed {
base: transition_terminal_final_task_base(committed.task().base().clone(), FinalTaskStatus::Completed, None).unwrap(),
result: serde_json::from_value(serde_json::json!({"content": [{"type": "text", "text": "completed by worker"}]})).unwrap(),
},
AtomicTaskCommitPhase::RequireInput => unreachable!(),
};
worker_for_probe
.persist_fenced_handoff_transition_clearing_input(
&committed, OWNER, fence, false, successor,
)
.unwrap();
}
AtomicTaskCommitPhase::RequireInput => {
worker_for_probe
.update_task(
&task_id,
&serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.unwrap(),
)
.unwrap();
}
}
*after_worker.lock().unwrap() = Some((
task_id.clone(),
input_key_store_snapshot(&worker_inner, &task_id),
));
if let Some(mutation) = mutation {
match mutation {
AtomicTaskReceiptMutation::Principal => {
committed.authenticated_principal =
Some(Sha256Digest::from_bytes([83; 32]));
}
AtomicTaskReceiptMutation::Generation => {
committed.generation = generation_for_probe.load(AtomicOrdering::SeqCst);
}
AtomicTaskReceiptMutation::OlderGeneration => {
let expected = generation_for_probe.load(AtomicOrdering::SeqCst);
assert!(expected > 1, "rollback uses a valid earlier generation");
committed.generation = expected - 1;
}
AtomicTaskReceiptMutation::Identity => match &mut committed.task {
FinalTask::Working(base) | FinalTask::InputRequired { base, .. } => {
base.task_id = FinalTaskId::parse("substituted-receipt").unwrap();
}
_ => unreachable!(),
},
AtomicTaskReceiptMutation::Payload => match &mut committed.task {
FinalTask::Working(base) => {
base.status_message = Some("substituted receipt".to_owned());
}
FinalTask::InputRequired { input_requests, .. } => {
let roots = input_requests.remove("roots").unwrap();
input_requests.insert("substituted-input-key".to_owned(), roots);
}
_ => unreachable!(),
},
}
}
committed
});
let runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.unwrap();
let service_cx = Cx::for_testing();
let _running_service = enter_task_service_runner(runner, &service_cx);
let result = match phase {
AtomicTaskCommitPhase::Create => {
*probe.after_commit.lock().unwrap() = Some(hook);
runtime
.create_task_for_request(&ctx, final_test_work_descriptor(), None)
.map(|_| ())
}
AtomicTaskCommitPhase::Update | AtomicTaskCommitPhase::RequireInput => {
let initial = final_working_task_with_ttl("atomic-commit-existing", 60_000);
let task_id = initial.base().task_id.clone();
let notification = final_task_notification(&initial);
let initial = if let Some(principal) = principal {
inner
.create_task_with_authenticated_work(
initial,
notification,
final_test_work_descriptor(),
principal,
)
.unwrap()
} else {
inner
.create_task_with_work(initial, notification, final_test_work_descriptor())
.unwrap()
};
match phase {
AtomicTaskCommitPhase::Update => {
worker
.require_input(&task_id, final_roots_request(), None)
.unwrap();
predecessor_generation.store(
inner
.get_task_snapshot(&task_id)
.unwrap()
.unwrap()
.generation(),
AtomicOrdering::SeqCst,
);
notifications.lock().unwrap().clear();
*probe.after_commit.lock().unwrap() = Some(hook);
let mut parameters = final_task_method_parameters(&task_id);
parameters["inputResponses"] = serde_json::json!({"roots": {"roots": []}});
dispatch_final_tasks_update(&runtime, &ctx, parameters).map(|_| ())
}
AtomicTaskCommitPhase::RequireInput => {
assert!(
inner
.take_initial_work_handoff_for_owner_if_current(&initial, OWNER)
.unwrap()
.is_some()
);
let fence = inner
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
initial.generation(),
OWNER,
)
.unwrap()
.unwrap();
predecessor_generation.store(initial.generation(), AtomicOrdering::SeqCst);
*probe.after_commit.lock().unwrap() = Some(hook);
let awaiting = FinalTask::InputRequired {
base: transition_final_task_base(
initial.task().base().clone(),
FinalTaskStatus::InputRequired,
None,
)
.unwrap(),
input_requests: final_roots_request(),
};
runtime.persist_fenced_handoff_transition_clearing_input(
&initial, OWNER, fence, false, awaiting,
)
}
AtomicTaskCommitPhase::Create => unreachable!(),
}
}
};
if mutation.is_some() {
assert_eq!(
result
.expect_err("a changed receipt cannot authorize acknowledgement")
.code,
fastmcp_core::McpErrorCode::InternalError
);
} else {
result.expect("a committed operation succeeds even after a worker advances its task");
}
let (task_id, snapshot) = retained_after_worker
.lock()
.unwrap()
.clone()
.expect("independent worker ran");
assert_eq!(
input_key_store_snapshot(&inner, &task_id),
snapshot,
"the request never rewrites its worker's later durable state"
);
assert_eq!(
inner
.get_task_snapshot(&task_id)
.unwrap()
.unwrap()
.authenticated_principal(),
principal
);
let notifications = notifications.lock().unwrap();
assert_eq!(
notifications.len(),
1,
"the already-obsolete predecessor notification is not emitted"
);
assert!(
final_tasks_match_exactly(
¬ifications[0].params.task,
&inner.get_task(&task_id).unwrap().unwrap()
)
.unwrap()
);
}
#[test]
fn task_03_atomic_create_receipt_survives_immediate_worker_progress() {
for authenticated in [false, true] {
assert_atomic_task_receipt(AtomicTaskCommitPhase::Create, authenticated, None);
}
}
#[test]
fn task_03_atomic_update_receipt_survives_immediate_worker_completion() {
for authenticated in [false, true] {
assert_atomic_task_receipt(AtomicTaskCommitPhase::Update, authenticated, None);
}
}
#[test]
fn task_03_atomic_input_required_receipt_survives_immediate_response() {
for authenticated in [false, true] {
assert_atomic_task_receipt(AtomicTaskCommitPhase::RequireInput, authenticated, None);
}
}
#[test]
fn task_03_atomic_receipts_reject_substitution_without_rewriting_successors() {
for phase in [
AtomicTaskCommitPhase::Create,
AtomicTaskCommitPhase::Update,
AtomicTaskCommitPhase::RequireInput,
] {
for mutation in [
AtomicTaskReceiptMutation::Identity,
AtomicTaskReceiptMutation::Payload,
AtomicTaskReceiptMutation::Principal,
] {
assert_atomic_task_receipt(phase, true, Some(mutation));
}
if !matches!(phase, AtomicTaskCommitPhase::Create) {
assert_atomic_task_receipt(
phase,
true,
Some(AtomicTaskReceiptMutation::Generation),
);
}
if matches!(phase, AtomicTaskCommitPhase::Update) {
assert_atomic_task_receipt(
phase,
true,
Some(AtomicTaskReceiptMutation::OlderGeneration),
);
}
}
}
#[test]
fn task_03_atomic_commit_survives_notification_observation_failure() {
for fail_observation in [false, true] {
let inner = Arc::new(InMemoryFinalTaskStore::default());
let notifications = Arc::new(AtomicUsize::new(0));
let observed = notifications.clone();
let probe = Arc::new(CommittedSnapshotProbeStore {
inner: inner.clone(),
after_commit: Mutex::new(None),
fail_observation: AtomicBool::new(fail_observation),
});
let runtime = FinalTaskRuntime::new(
probe,
FinalTaskRuntimeConfig::new(60_000, None).unwrap(),
Arc::new(move |_| {
observed.fetch_add(1, AtomicOrdering::SeqCst);
}),
);
let runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.unwrap();
let cx = Cx::for_testing();
let _service = enter_task_service_runner(runner, &cx);
let created = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("notification observation cannot revoke the successfully committed task");
assert!(
inner
.get_task(&created.task.base().task_id)
.unwrap()
.is_some()
);
assert_eq!(
notifications.load(AtomicOrdering::SeqCst),
usize::from(!fail_observation)
);
}
}
struct CancellationReadbackProbeStore {
inner: Arc<InMemoryFinalTaskStore>,
retire_after_intent_read: bool,
retirement: Mutex<Option<Box<dyn FnOnce() + Send>>>,
}
impl FinalTaskStore for CancellationReadbackProbeStore {
delegate_final_execution_policy!();
fn create_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.create_task(task, notification)
}
fn get_task(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTask>> {
self.inner.get_task(task_id)
}
fn get_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.get_task_snapshot(task_id)
}
fn replace_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.replace_task(task, notification)
}
fn replace_task_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<bool> {
self.inner
.replace_task_if_current(expected, task, notification)
}
fn request_cancellation_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
cancelled_task: FinalTask,
cancelled_notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.request_cancellation_and_clear_input_if_current(
expected,
cancelled_task,
cancelled_notification,
)
}
fn request_cancellation(&self, task_id: &FinalTaskId) -> McpResult<()> {
self.inner.request_cancellation(task_id)
}
fn request_cancellation_if_current(&self, expected: &FinalTaskSnapshot) -> McpResult<bool> {
self.inner.request_cancellation_if_current(expected)
}
fn is_cancellation_requested(&self, task_id: &FinalTaskId) -> McpResult<bool> {
let retirement = self.retirement.lock().unwrap().take();
if self.retire_after_intent_read {
let requested = self.inner.is_cancellation_requested(task_id)?;
if let Some(retire) = retirement {
retire();
}
Ok(requested)
} else {
if let Some(retire) = retirement {
retire();
}
self.inner.is_cancellation_requested(task_id)
}
}
fn retention_clock_now(&self) -> Instant {
self.inner.retention_clock_now()
}
fn task_retention_deadline_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>> {
self.inner
.task_retention_deadline_if_current(task_id, generation)
}
}
#[derive(Clone, Copy, Debug)]
enum CancellationReadbackMutation {
Principal,
Identity,
CreatedAt,
Retention,
PollInterval,
SameGeneration,
OlderGeneration,
Completed,
WorkingWithoutIntent,
ActiveSuccessor,
}
fn assert_cancellation_retirement_readback(
retire_after_intent_read: bool,
mutation: Option<CancellationReadbackMutation>,
) {
const OWNER: &str = "cancellation-readback-owner";
let inner = Arc::new(InMemoryFinalTaskStore::default());
let worker_runtime = final_task_runtime(inner.clone(), Arc::new(AtomicBool::new(false)));
let responses = serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed roots response");
let task_id = create_accepted_final_input(&worker_runtime, responses);
let expected = inner.get_task_snapshot(&task_id).unwrap().unwrap();
assert!(
inner
.take_input_handoff_for_owner_if_current(&expected, OWNER)
.unwrap()
.is_some()
);
let fence = inner
.begin_handoff_dispatch_for_owner_if_current(&task_id, expected.generation(), OWNER)
.unwrap()
.expect("the resumed supervisor owns the dispatch fence");
let retained_after_retirement = Arc::new(Mutex::new(None));
let retained_for_worker = retained_after_retirement.clone();
let inner_for_worker = inner.clone();
let expected_for_worker = expected.clone();
let task_for_worker = task_id.clone();
let retirement = Box::new(move || {
let cancelled = FinalTask::Cancelled(
transition_terminal_final_task_base(
expected_for_worker.task().base().clone(),
FinalTaskStatus::Cancelled,
Some("retired by the elected supervisor".to_owned()),
)
.unwrap(),
);
worker_runtime
.persist_fenced_handoff_transition_clearing_input(
&expected_for_worker,
OWNER,
fence,
true,
cancelled.clone(),
)
.expect("the cancellation winner lets its exact elected worker retire");
if let Some(mutation) = mutation {
let mut state = inner_for_worker.state.lock().unwrap();
let mut base = cancelled.base().clone();
match mutation {
CancellationReadbackMutation::Principal => {
state
.authenticated_principals
.insert(task_for_worker.clone(), Sha256Digest::from_bytes([17; 32]));
}
CancellationReadbackMutation::Identity => {
base.task_id = FinalTaskId::parse("substituted-task").unwrap();
}
CancellationReadbackMutation::CreatedAt => {
base.created_at =
FinalTaskTimestamp::parse("2026-07-28T12:00:00Z").unwrap();
}
CancellationReadbackMutation::Retention => {
base.ttl_ms = None;
}
CancellationReadbackMutation::PollInterval => {
base.poll_interval_ms = Some(final_task_duration(9_999).unwrap());
}
CancellationReadbackMutation::SameGeneration => {
state
.generations
.insert(task_for_worker.clone(), expected_for_worker.generation());
}
CancellationReadbackMutation::OlderGeneration => {
state.generations.insert(
task_for_worker.clone(),
expected_for_worker.generation() - 1,
);
}
CancellationReadbackMutation::Completed => {
base.status = FinalTaskStatus::Completed;
}
CancellationReadbackMutation::WorkingWithoutIntent => {
base = expected_for_worker.task().base().clone();
state
.generations
.insert(task_for_worker.clone(), expected_for_worker.generation());
}
CancellationReadbackMutation::ActiveSuccessor => {
base.status = FinalTaskStatus::Working;
state.cancellation_requests.insert(task_for_worker.clone());
}
}
let replacement = match mutation {
CancellationReadbackMutation::Completed => FinalTask::Completed {
base,
result: serde_json::from_value(serde_json::json!({"content": []})).unwrap(),
},
CancellationReadbackMutation::WorkingWithoutIntent
| CancellationReadbackMutation::ActiveSuccessor => FinalTask::Working(base),
_ => FinalTask::Cancelled(base),
};
state.tasks.insert(task_for_worker.clone(), replacement);
}
*retained_for_worker.lock().unwrap() = Some(input_key_store_snapshot(
&inner_for_worker,
&task_for_worker,
));
});
let store = Arc::new(CancellationReadbackProbeStore {
inner: inner.clone(),
retire_after_intent_read,
retirement: Mutex::new(Some(retirement)),
});
let request_notifications = Arc::new(AtomicUsize::new(0));
let notifications_for_runtime = request_notifications.clone();
let runtime = FinalTaskRuntime::new(
store,
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).unwrap(),
Arc::new(move |_| {
notifications_for_runtime.fetch_add(1, AtomicOrdering::SeqCst);
}),
);
let result = dispatch_final_tasks_cancel(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
final_task_method_parameters(&task_id),
);
if let Some(mutation) = mutation {
let error = result.expect_err("a substituted cancellation successor must fail closed");
assert_eq!(
error.code,
fastmcp_core::McpErrorCode::InternalError,
"{mutation:?}"
);
} else {
assert_eq!(
result.expect("successful cancellation survives prompt worker retirement"),
serde_json::json!({"resultType": "complete"})
);
let current = inner.get_task_snapshot(&task_id).unwrap().unwrap();
assert!(matches!(current.task(), FinalTask::Cancelled(_)));
assert!(current.generation() > expected.generation());
assert!(!inner.is_cancellation_requested(&task_id).unwrap());
assert_eq!(
current.task().base().status_message.as_deref(),
Some("retired by the elected supervisor")
);
}
assert_eq!(
input_key_store_snapshot(&inner, &task_id),
retained_after_retirement
.lock()
.unwrap()
.clone()
.expect("worker retirement ran"),
"acknowledgement or rejection leaves the worker's durable state unchanged"
);
assert_eq!(
request_notifications.load(AtomicOrdering::SeqCst),
0,
"only the worker emits its terminal notification"
);
}
#[test]
fn task_03_cancellation_ack_survives_retirement_before_intent_read() {
assert_cancellation_retirement_readback(false, None);
}
#[test]
fn task_03_cancellation_ack_survives_retirement_between_intent_and_snapshot_reads() {
assert_cancellation_retirement_readback(true, None);
}
#[test]
fn task_03_cancellation_readback_rejects_substituted_successors_without_mutation() {
for mutation in [
CancellationReadbackMutation::Principal,
CancellationReadbackMutation::Identity,
CancellationReadbackMutation::CreatedAt,
CancellationReadbackMutation::Retention,
CancellationReadbackMutation::PollInterval,
CancellationReadbackMutation::SameGeneration,
CancellationReadbackMutation::OlderGeneration,
CancellationReadbackMutation::Completed,
CancellationReadbackMutation::WorkingWithoutIntent,
CancellationReadbackMutation::ActiveSuccessor,
] {
assert_cancellation_retirement_readback(false, Some(mutation));
}
}
#[test]
fn task_03_cancellation_rejects_terminal_generation_rollback_without_mutation() {
for advances in [true, false] {
let working = final_working_task_without_ttl("terminal-cancellation-generation");
let store = Arc::new(RuntimeBoundaryProbeFinalTaskStore::new(working));
let expected = store.snapshot();
let cancelled = FinalTask::Cancelled(
transition_terminal_final_task_base(
expected.task().base().clone(),
FinalTaskStatus::Cancelled,
None,
)
.unwrap(),
);
let generation = if advances {
expected.generation() + 1
} else {
expected.generation() - 1
};
let returned = FinalTaskSnapshot::new(cancelled.clone(), generation);
*store.snapshot.lock().unwrap() = returned.clone();
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).unwrap(),
Arc::new(|_| {}),
);
let result =
runtime.validate_cancellation_store_result(&expected, &cancelled, returned);
if advances {
assert!(
result.expect("an exact retained terminal cancellation advances generation")
);
} else {
let error = result.expect_err(
"changing only the committed generation to an older value must be rejected",
);
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
}
let retained = store.snapshot();
assert_eq!(retained.generation(), generation);
assert!(final_tasks_match_exactly(retained.task(), &cancelled).unwrap());
assert_eq!(store.transition_write_calls.load(AtomicOrdering::SeqCst), 0);
}
}
#[test]
fn task_03_runtime_rejects_custom_store_cancellation_substitution_and_false_cas_unchanged() {
let initial = final_working_task_without_ttl("task-runtime-custom-cancel");
let task_id = initial.base().task_id.clone();
let store = Arc::new(RuntimeBoundaryProbeFinalTaskStore::new(initial));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| {}),
);
let before = store.snapshot();
let FinalTask::Working(base) = before.task().clone() else {
unreachable!("the probe begins working");
};
let substituted = FinalTask::Cancelled(
transition_terminal_final_task_base(
base,
FinalTaskStatus::Cancelled,
Some("substituted cancellation".to_owned()),
)
.expect("fixed cancellation transition"),
);
*store
.cancellation_result_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(FinalTaskSnapshot::new(
substituted,
before.generation().saturating_add(1),
));
let error = runtime
.cancel_task(&task_id)
.expect_err("a near-identical cancelled task cannot substitute the runtime intent");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
let FinalTask::Working(mut substituted_active_base) = before.task().clone() else {
unreachable!("the probe remains working after rejected terminal substitution");
};
substituted_active_base.status_message = Some("substituted active cancellation".to_owned());
*store
.cancellation_result_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(FinalTaskSnapshot::new(
FinalTask::Working(substituted_active_base),
before.generation(),
));
let error = runtime
.cancel_task(&task_id)
.expect_err("an active cancellation result must retain the exact expected snapshot");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
*store
.cancellation_result_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
store.force_false_cas.store(true, AtomicOrdering::SeqCst);
let FinalTask::Working(mut terminal_base) = before.task().clone() else {
unreachable!("the probe remains working after rejected substitution");
};
terminal_base.status_message = Some("terminal candidate".to_owned());
let error = runtime
.persist_transition_clearing_input(&before, FinalTask::Working(terminal_base))
.expect_err("a false compare-and-swap is not an accepted terminal transition");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
let after = store.snapshot();
assert!(
final_tasks_match_exactly(before.task(), after.task()).expect("compare probe task"),
"substitution and false-CAS rejection leave the durable task unchanged"
);
assert_eq!(after.generation(), before.generation());
}
#[test]
fn task_03_runtime_rejects_custom_store_terminal_transition_substitution() {
let initial = final_working_task_without_ttl("task-runtime-custom-terminal");
let task_id = initial.base().task_id.clone();
let store = Arc::new(RuntimeBoundaryProbeFinalTaskStore::new(initial));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| {}),
);
let before = store.snapshot();
let FinalTask::Working(base) = before.task().clone() else {
unreachable!("the probe begins working");
};
let intended = FinalTask::Cancelled(
transition_terminal_final_task_base(
base.clone(),
FinalTaskStatus::Cancelled,
Some("intended terminal transition".to_owned()),
)
.expect("fixed terminal transition is valid"),
);
let substituted = FinalTask::Cancelled(
transition_terminal_final_task_base(
base,
FinalTaskStatus::Cancelled,
Some("substituted terminal transition".to_owned()),
)
.expect("fixed substituted terminal transition is valid"),
);
*store
.transition_result_override
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(substituted);
let error = runtime
.persist_transition_clearing_input(&before, intended)
.expect_err("a store may not substitute a near-identical terminal transition");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
let after = store.snapshot();
assert!(
!final_tasks_match_exactly(before.task(), after.task()).expect("compare probe task"),
"the deliberately permissive store records its substituted write for boundary detection"
);
assert_eq!(after.generation(), before.generation().saturating_add(1));
assert_eq!(after.task().base().task_id, task_id);
}
#[test]
fn task_03_in_memory_false_cas_does_not_reclaim_expired_retained_task() {
let (store, now) = in_memory_store_with_test_clock(4);
let task =
final_working_task_with_wire_durations("task-false-cas-retention", Some("1"), None);
let task_id = task.base().task_id.clone();
let notification = final_task_notification(&task);
store
.create_task_with_work(task.clone(), notification, final_test_work_descriptor())
.expect("bounded retained task is created");
let current = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.generations
.get(&task_id)
.copied()
.expect("created task retains a generation");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) += StdDuration::from_millis(2);
let stale = FinalTaskSnapshot::new(task.clone(), current.saturating_add(1));
assert!(
!store
.replace_task_if_current(&stale, task.clone(), final_task_notification(&task))
.expect("stale compare-and-swap returns false"),
"changing only the expected generation loses the CAS"
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
state.tasks.contains_key(&task_id) && state.expires_at.contains_key(&task_id),
"a false CAS must not use expiry cleanup to delete retained task state"
);
drop(state);
let empty_roots: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed empty roots response");
assert!(
store
.replace_task_and_append_input_if_current(
&stale,
task.clone(),
final_task_notification(&task),
empty_roots,
)
.expect("stale append compare-and-swap returns no committed snapshot")
.is_none(),
"stale append compare-and-swap must preserve retained task state"
);
assert!(
store
.replace_task_and_clear_input_if_current(
&stale,
task.clone(),
final_task_notification(&task),
)
.expect("stale clear compare-and-swap returns no committed snapshot")
.is_none(),
"stale clear compare-and-swap must preserve retained task state"
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
final_tasks_match_exactly(
&task,
state.tasks.get(&task_id).expect("stale CAS retains task"),
)
.expect("compare retained task with original"),
"all stale replacement CAS variants retain the original task"
);
assert_eq!(
state.generations.get(&task_id),
Some(¤t),
"all stale replacement CAS variants retain the current generation"
);
}
#[test]
fn task_03_in_memory_runtime_capacity_one_variable_rejection() {
let runtime = FinalTaskRuntime::in_memory_with_capacity(
1,
FinalTaskRuntimeConfig::new(60_000, None).expect("valid in-memory task policy"),
Arc::new(|_| {}),
)
.expect("positive capacity constructs the in-memory runtime");
let service_runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("installing a caller-owned service reserves the runner");
let service_cx = Cx::for_testing();
let _running_service = enter_task_service_runner(service_runner, &service_cx);
let first = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("first task fits the one-task capacity");
let first_id = first.task.base().task_id.clone();
assert!(
runtime
.create_task_with_work(final_test_work_descriptor(), None)
.is_err(),
"only the second create changes from the admitted one-task baseline"
);
assert!(matches!(
runtime
.get_task(&first_id)
.expect("rejected second create preserves the first task")
.task,
FinalTask::Working(_)
));
}
#[test]
fn task_02_final_payload_counter_counts_escaping_and_rejects_overflow() {
let value = serde_json::json!({"text": "quoted \"line\"\n雪", "items": [null, true, 1]});
let bytes = encoded_final_task_test_bytes(&value);
assert_eq!(
final_task_payload_bytes(&value, bytes, "fixture").unwrap(),
bytes
);
assert!(final_task_payload_bytes(&value, bytes - 1, "fixture").is_err());
let mut counter = FinalTaskPayloadByteCounter {
bytes: usize::MAX,
limit: usize::MAX,
};
assert!(std::io::Write::write(&mut counter, b"x").is_err());
assert_eq!(counter.bytes, usize::MAX);
assert!(InMemoryFinalTaskStore::with_payload_capacity(1, 0).is_err());
assert!(
InMemoryFinalTaskStore::with_payload_capacity(
1,
InMemoryFinalTaskStore::HARD_MAX_PAYLOAD_BYTES + 1,
)
.is_err()
);
assert_eq!(
InMemoryFinalTaskStore::with_payload_capacity(
1,
InMemoryFinalTaskStore::HARD_MAX_PAYLOAD_BYTES,
)
.unwrap()
.max_payload_bytes(),
InMemoryFinalTaskStore::HARD_MAX_PAYLOAD_BYTES,
);
}
#[test]
fn task_02_final_descriptor_payload_boundary_preserves_create_state() {
for authenticated in [false, true] {
let store = InMemoryFinalTaskStore::new(2).unwrap();
let task = final_working_task_with_ttl("task-descriptor-byte-boundary", 60_000);
let task_id = task.base().task_id.clone();
let before = final_task_restoration_snapshot(&store, &task_id);
for excess in [1, 0] {
let descriptor = FinalTaskWorkDescriptor::new(serde_json::Value::String(
"x".repeat(MAX_FINAL_TASK_DESCRIPTOR_BYTES - 2 + excess),
))
.unwrap();
let result = if authenticated {
store.create_task_with_authenticated_work(
task.clone(),
final_task_notification(&task),
descriptor,
Sha256Digest::from_bytes([11; 32]),
)
} else {
store.create_task_with_work(
task.clone(),
final_task_notification(&task),
descriptor,
)
};
if excess == 1 {
assert!(result.is_err());
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
assert_eq!(store.retained_payload_bytes(), 0);
} else {
result.expect("an exactly bounded descriptor is accepted after rejection");
assert!(store.get_task(&task_id).unwrap().is_some());
assert_final_task_payload_accounting(&store);
}
}
}
}
#[test]
fn task_02_final_metadata_payload_boundary_preserves_create_state() {
let store = InMemoryFinalTaskStore::new(1).unwrap();
let FinalTask::Working(mut base) = final_working_task_without_ttl("task-metadata-boundary")
else {
unreachable!();
};
base.status_message = Some(String::new());
base.last_updated_at = FinalTaskTimestamp::parse("2026-07-28T12:00:00Z").unwrap();
let overhead = encoded_final_task_test_bytes(&base);
let metadata_limit =
MAX_FINAL_TASK_METADATA_BYTES - FINAL_TASK_CANCELLATION_METADATA_RESERVE_BYTES;
let before = final_task_restoration_snapshot(&store, &base.task_id);
for excess in [1, 0] {
base.status_message = Some("x".repeat(metadata_limit - overhead + excess));
let task = FinalTask::Working(base.clone());
let result = store.create_task(task.clone(), final_task_notification(&task));
if excess == 1 {
assert!(result.is_err());
assert_eq!(
final_task_restoration_snapshot(&store, &base.task_id),
before
);
} else {
result.expect("exactly bounded metadata creates through the bare store path");
assert_final_task_payload_accounting(&store);
}
}
let snapshot = store.get_task_snapshot(&base.task_id).unwrap().unwrap();
base.status = FinalTaskStatus::Cancelled;
base.last_updated_at =
FinalTaskTimestamp::parse("2026-07-28T12:00:00.123456789+00:00").unwrap();
assert_eq!(
encoded_final_task_test_bytes(&base),
MAX_FINAL_TASK_METADATA_BYTES,
);
let cancelled = FinalTask::Cancelled(base);
assert!(
store
.request_cancellation_and_clear_input_if_current(
&snapshot,
cancelled.clone(),
final_task_notification(&cancelled),
)
.unwrap()
.is_some()
);
assert_final_task_payload_accounting(&store);
}
#[test]
fn task_02_final_partial_input_payload_aggregate_is_atomic() {
let (store, _now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let requests: FinalTaskInputRequests = serde_json::from_value(serde_json::json!({
"left": {"method": "roots/list"}, "right": {"method": "roots/list"}
}))
.unwrap();
runtime.require_input(&task_id, requests, None).unwrap();
let right_requests: FinalTaskInputRequests = serde_json::from_value(serde_json::json!({
"right": {"method": "roots/list"}
}))
.unwrap();
let combined_overflow = final_roots_input_with_encoded_bytes(
"left",
MAX_FINAL_TASK_INPUT_BYTES - encoded_final_task_test_bytes(&right_requests) + 1,
);
let before_partial = final_task_restoration_snapshot(&store, &task_id);
assert!(runtime.update_task(&task_id, &combined_overflow).is_err());
assert_eq!(
final_task_restoration_snapshot(&store, &task_id),
before_partial
);
let left = final_roots_input_with_encoded_bytes("left", MAX_FINAL_TASK_INPUT_BYTES - 512);
runtime
.update_task(&task_id, &left)
.expect("partial input is retained");
assert!(matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::InputRequired { .. })
));
assert_final_task_payload_accounting(&store);
let before = final_task_restoration_snapshot(&store, &task_id);
let oversized = final_roots_input_with_encoded_bytes("right", 514);
assert!(runtime.update_task(&task_id, &oversized).is_err());
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
let right = final_roots_input_with_encoded_bytes("right", 513);
runtime
.update_task(&task_id, &right)
.expect("exactly bounded merged input resumes work");
assert!(matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::Working(_))
));
assert_final_task_payload_accounting(&store);
let retained = store.state.lock().unwrap().accepted_inputs[&task_id].clone();
assert_eq!(
encoded_final_task_test_bytes(&retained),
MAX_FINAL_TASK_INPUT_BYTES
);
}
#[test]
fn task_02_final_outstanding_input_payload_boundary_preserves_work() {
let (store, _now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let mut wire = serde_json::json!({
"sample": {
"method": "sampling/createMessage",
"params": {"messages": [], "maxTokens": 16, "systemPrompt": ""}
}
});
let empty: FinalTaskInputRequests = serde_json::from_value(wire.clone()).unwrap();
let overhead = encoded_final_task_test_bytes(&empty);
let before = final_task_restoration_snapshot(&store, &task_id);
for excess in [1, 0] {
wire["sample"]["params"]["systemPrompt"] = serde_json::Value::String(
"x".repeat(MAX_FINAL_TASK_INPUT_BYTES - overhead + excess),
);
let requests: FinalTaskInputRequests = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(
encoded_final_task_test_bytes(&requests),
MAX_FINAL_TASK_INPUT_BYTES + excess,
);
let result = runtime.require_input(&task_id, requests, None);
if excess == 1 {
assert!(result.is_err());
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
} else {
assert!(matches!(result.unwrap(), FinalTask::InputRequired { .. }));
assert_final_task_payload_accounting(&store);
}
}
}
#[test]
fn task_02_final_terminal_payload_boundaries_preserve_fenced_state() {
for failed in [false, true] {
let (store, _now) = in_memory_store_with_test_clock(1);
let task = final_working_task_with_ttl("task-result-byte-boundary", 60_000);
let task_id = task.base().task_id.clone();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.unwrap();
let snapshot = store.get_task_snapshot(&task_id).unwrap().unwrap();
store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "result-owner")
.unwrap()
.unwrap();
let fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
snapshot.generation(),
"result-owner",
)
.unwrap()
.unwrap();
let before = final_task_restoration_snapshot(&store, &task_id);
let empty = if failed {
serde_json::json!({"code": -32000, "message": ""})
} else {
serde_json::json!({"content": [{"type": "text", "text": ""}]})
};
let overhead = encoded_final_task_test_bytes(&empty);
for excess in [1, 0] {
let mut wire = empty.clone();
let padding = serde_json::Value::String(
"x".repeat(MAX_FINAL_TASK_RESULT_BYTES - overhead + excess),
);
let replacement = if failed {
wire["message"] = padding;
FinalTask::Failed {
base: transition_terminal_final_task_base(
task.base().clone(),
FinalTaskStatus::Failed,
None,
)
.unwrap(),
error: serde_json::from_value(wire).unwrap(),
}
} else {
wire["content"][0]["text"] = padding;
FinalTask::Completed {
base: transition_terminal_final_task_base(
task.base().clone(),
FinalTaskStatus::Completed,
None,
)
.unwrap(),
result: serde_json::from_value(wire).unwrap(),
}
};
let result = store.replace_task_and_clear_input_for_handoff_if_current(
&snapshot,
"result-owner",
fence,
false,
replacement.clone(),
final_task_notification(&replacement),
);
if excess == 1 {
assert!(result.is_err());
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
} else {
assert!(
result
.expect("exact terminal member is committed by its fenced owner")
.is_some()
);
assert!(store.retained_payload_bytes() >= 2 * MAX_FINAL_TASK_RESULT_BYTES);
assert_final_task_payload_accounting(&store);
}
}
}
}
#[test]
fn task_02_final_payload_capacity_charges_notifications_deltas_and_terminal_retention() {
let task = final_working_task_with_ttl("task-payload-capacity", 1_000);
let task_id = task.base().task_id.clone();
let descriptor = final_test_work_descriptor();
let descriptor_bytes = encoded_final_task_test_bytes(descriptor.as_value());
let FinalTask::Working(mut grown_base) = task.clone() else {
unreachable!()
};
grown_base.status_message = Some("x".repeat(2_048));
let grown = FinalTask::Working(grown_base.clone());
let limit = encoded_final_task_test_bytes(&grown)
+ encoded_final_task_test_bytes(&final_task_notification(&grown))
+ 2 * descriptor_bytes
+ FINAL_TASK_CANCELLATION_RESERVE_BYTES;
let insufficient = InMemoryFinalTaskStore::with_payload_capacity(2, limit - 1).unwrap();
let before = final_task_restoration_snapshot(&insufficient, &task_id);
assert!(
insufficient
.create_task_with_work(
grown.clone(),
final_task_notification(&grown),
descriptor.clone(),
)
.is_err()
);
assert_eq!(
final_task_restoration_snapshot(&insufficient, &task_id),
before
);
let now = Arc::new(Mutex::new(Instant::now()));
let clock = Arc::clone(&now);
let store = InMemoryFinalTaskStore::with_clock_and_payload_capacity(
2,
limit,
Arc::new(move || *clock.lock().unwrap()),
)
.unwrap();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
descriptor.clone(),
)
.unwrap();
let initial = store.get_task_snapshot(&task_id).unwrap().unwrap();
assert!(
store
.replace_task_if_current(&initial, grown.clone(), final_task_notification(&grown))
.unwrap()
);
assert_eq!(
store.retained_payload_bytes() + FINAL_TASK_CANCELLATION_RESERVE_BYTES,
limit
);
assert_final_task_payload_accounting(&store);
let current = store.get_task_snapshot(&task_id).unwrap().unwrap();
let before = final_task_restoration_snapshot(&store, &task_id);
grown_base.status_message.as_mut().unwrap().push('x');
let oversized = FinalTask::Working(grown_base);
assert!(
store
.replace_task_if_current(
¤t,
oversized.clone(),
final_task_notification(&oversized),
)
.is_err()
);
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
let other = final_working_task_with_ttl("task-payload-sibling", 1_000);
assert!(
store
.create_task(other.clone(), final_task_notification(&other))
.is_err()
);
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
let completed = FinalTask::Completed {
base: transition_terminal_final_task_base(
task.base().clone(),
FinalTaskStatus::Completed,
None,
)
.unwrap(),
result: serde_json::from_value(serde_json::json!({"content": []})).unwrap(),
};
assert!(
store
.replace_task_if_current(
¤t,
completed.clone(),
final_task_notification(&completed),
)
.unwrap()
);
let terminal_bytes = encoded_final_task_test_bytes(&completed)
+ encoded_final_task_test_bytes(&final_task_notification(&completed))
+ descriptor_bytes;
assert_eq!(store.retained_payload_bytes(), terminal_bytes);
store
.create_task(other.clone(), final_task_notification(&other))
.expect("shrinking a replacement releases aggregate capacity");
assert_final_task_payload_accounting(&store);
assert!(store.retained_payload_bytes() > terminal_bytes);
*now.lock().unwrap() += StdDuration::from_millis(1_000);
assert_eq!(store.retained_payload_bytes(), 0);
assert_eq!(
store.retained_payload_bytes(),
0,
"expiry releases each charge once"
);
assert_eq!(store.task_count(), 0);
assert_final_task_payload_accounting(&store);
store
.create_task_with_work(grown.clone(), final_task_notification(&grown), descriptor)
.expect("expiry allows the full encoded capacity to be reused");
assert_eq!(
store.retained_payload_bytes() + FINAL_TASK_CANCELLATION_RESERVE_BYTES,
limit
);
assert_final_task_payload_accounting(&store);
}
#[test]
fn task_02_final_handoff_completion_and_cancellation_release_only_consumed_payload() {
for resumed in [false, true] {
for cancel in [false, true] {
let (store, _now) = in_memory_store_with_test_clock(1);
let runtime =
final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let (task_id, released) = if resumed {
let inputs = final_roots_input_with_encoded_bytes("roots", 512);
(create_accepted_final_input(&runtime, inputs), 512)
} else {
let created = create_final_task_state_fixture(&runtime, None);
(
created.task.base().task_id.clone(),
encoded_final_task_test_bytes(final_test_work_descriptor().as_value()),
)
};
let snapshot = store.get_task_snapshot(&task_id).unwrap().unwrap();
if resumed {
store
.take_input_handoff_for_owner_if_current(&snapshot, "payload-owner")
.unwrap()
.unwrap();
} else {
store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "payload-owner")
.unwrap()
.unwrap();
}
let fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
snapshot.generation(),
"payload-owner",
)
.unwrap()
.unwrap();
let before = store.retained_payload_bytes();
if cancel {
assert!(store.request_cancellation_if_current(&snapshot).unwrap());
assert!(store.request_cancellation_if_current(&snapshot).unwrap());
} else {
assert!(
store
.finish_handoff_dispatch_for_owner_if_current(
&task_id,
snapshot.generation(),
"payload-owner",
fence,
)
.unwrap()
);
assert!(
!store
.finish_handoff_dispatch_for_owner_if_current(
&task_id,
snapshot.generation(),
"payload-owner",
fence,
)
.unwrap()
);
}
assert_eq!(store.retained_payload_bytes(), before - released);
assert!(store.get_task(&task_id).unwrap().is_some());
assert!(
store
.state
.lock()
.unwrap()
.work_descriptors
.contains_key(&task_id)
);
assert_final_task_payload_accounting(&store);
}
}
}
#[test]
fn task_02_final_full_payload_capacity_preserves_cancellation_retirement() {
let mut task = final_working_task_with_ttl("task-full-cancel", 60_000);
if let FinalTask::Working(base) = &mut task {
base.last_updated_at = FinalTaskTimestamp::parse("2026-07-28T12:00:00Z").unwrap();
}
let task_id = task.base().task_id.clone();
let initial_bytes = encoded_final_task_test_bytes(&task)
+ encoded_final_task_test_bytes(&final_task_notification(&task));
for maximum_timestamp in [false, true] {
let store = Arc::new(
InMemoryFinalTaskStore::with_payload_capacity(
1,
initial_bytes + FINAL_TASK_CANCELLATION_RESERVE_BYTES,
)
.unwrap(),
);
store
.create_task(task.clone(), final_task_notification(&task))
.unwrap();
if maximum_timestamp {
let snapshot = store.get_task_snapshot(&task_id).unwrap().unwrap();
let mut base = task.base().clone();
base.status = FinalTaskStatus::Cancelled;
base.last_updated_at =
FinalTaskTimestamp::parse("2026-07-28T12:00:00.123456789+00:00").unwrap();
let cancelled = FinalTask::Cancelled(base);
assert!(
store
.request_cancellation_and_clear_input_if_current(
&snapshot,
cancelled.clone(),
final_task_notification(&cancelled),
)
.unwrap()
.is_some()
);
assert_eq!(store.retained_payload_bytes(), initial_bytes + 34);
} else {
let runtime =
final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
runtime
.cancel_task(&task_id)
.expect("cancellation can grow its control fields at capacity");
assert!(store.retained_payload_bytes() > initial_bytes);
}
assert!(matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::Cancelled(_))
));
assert_final_task_payload_accounting(&store);
}
for expiry_retirement in [false, true] {
let mut sibling = final_working_task_with_ttl("task-full-cancel-sibling", 60_000);
if let FinalTask::Working(base) = &mut sibling {
base.status_message = Some("x".repeat(256));
}
let sibling_id = sibling.base().task_id.clone();
let sibling_charge = encoded_final_task_test_bytes(&sibling)
+ encoded_final_task_test_bytes(&final_task_notification(&sibling))
+ FINAL_TASK_CANCELLATION_RESERVE_BYTES;
let descriptor = FinalTaskWorkDescriptor::new(serde_json::Value::String(
"x".repeat(sibling_charge - 2),
))
.unwrap();
let capacity =
initial_bytes + 2 * sibling_charge + FINAL_TASK_CANCELLATION_RESERVE_BYTES;
let now = Arc::new(Mutex::new(Instant::now()));
let clock = Arc::clone(&now);
let store = Arc::new(
InMemoryFinalTaskStore::with_clock_and_payload_capacity(
2,
capacity,
Arc::new(move || *clock.lock().unwrap()),
)
.unwrap(),
);
store
.create_task_with_work(task.clone(), final_task_notification(&task), descriptor)
.unwrap();
let snapshot = store.get_task_snapshot(&task_id).unwrap().unwrap();
store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "cancel-owner")
.unwrap()
.unwrap();
let fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
snapshot.generation(),
"cancel-owner",
)
.unwrap()
.unwrap();
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
runtime.cancel_task(&task_id).unwrap();
assert!(matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::Working(_))
));
store
.create_task(sibling.clone(), final_task_notification(&sibling))
.expect("the sibling may use the released initial descriptor bytes");
assert_eq!(
store.retained_payload_bytes() + 2 * FINAL_TASK_CANCELLATION_RESERVE_BYTES,
capacity,
);
let sibling_before = store.get_task_snapshot(&sibling_id).unwrap().unwrap();
let sibling_notification =
serde_json::to_value(store.latest_notification(&sibling_id)).unwrap();
let mut pressure = sibling.clone();
if let FinalTask::Working(base) = &mut pressure {
base.status_message.as_mut().unwrap().push('x');
}
assert!(
store
.replace_task_if_current(
&sibling_before,
pressure.clone(),
final_task_notification(&pressure),
)
.is_err(),
"a sibling cannot consume even two bytes of reserved cancellation capacity",
);
if expiry_retirement {
*now.lock().unwrap() += IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
} else {
let cancelled = FinalTask::Cancelled(
transition_terminal_final_task_base(
task.base().clone(),
FinalTaskStatus::Cancelled,
None,
)
.unwrap(),
);
assert!(
store
.replace_task_and_clear_input_for_handoff_if_current(
&snapshot,
"cancel-owner",
fence,
true,
cancelled.clone(),
final_task_notification(&cancelled),
)
.unwrap()
.is_some()
);
}
assert!(matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::Cancelled(_))
));
let sibling_after = store.get_task_snapshot(&sibling_id).unwrap().unwrap();
assert_eq!(sibling_before.generation(), sibling_after.generation());
assert_eq!(
serde_json::to_value(sibling_before.task()).unwrap(),
serde_json::to_value(sibling_after.task()).unwrap(),
);
assert_eq!(
serde_json::to_value(store.latest_notification(&sibling_id)).unwrap(),
sibling_notification,
);
assert_final_task_payload_accounting(&store);
}
}
#[test]
fn task_03_in_memory_store_positive_ttl_reclaims_capacity_at_deterministic_deadline() {
const TTL_MS: u64 = 60_000;
let (store, now) = in_memory_store_with_test_clock(1);
let first = final_working_task_with_ttl("task-positive-ttl-first", TTL_MS);
let first_id = first.base().task_id.clone();
store
.create_task(first.clone(), final_task_notification(&first))
.expect("first task fits the bounded store");
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(TTL_MS))
.expect("positive task TTL fits the monotonic test clock");
drop(clock);
let second = final_working_task_without_ttl("task-positive-ttl-second");
let second_id = second.base().task_id.clone();
store
.create_task(second.clone(), final_task_notification(&second))
.expect("the expired first task releases bounded capacity");
assert!(
store
.get_task(&first_id)
.expect("expired task lookup is readable")
.is_none()
);
assert!(store.latest_notification(&first_id).is_none());
assert_eq!(store.task_count(), 1);
assert!(
store
.get_task(&second_id)
.expect("replacement task lookup is readable")
.is_some()
);
}
#[test]
fn task_03_in_memory_store_rejects_unrepresentable_durations_before_state_mutation() {
const ONE_OVER_U64: &str = "18446744073709551616";
let (store, _) = in_memory_store_with_test_clock(2);
for (task_id, field) in [
("task-unrepresentable-ttl", "ttlMs"),
("task-unrepresentable-poll", "pollIntervalMs"),
] {
let task = final_working_task_without_ttl(task_id);
let task_id = task.base().task_id.clone();
let mut wire = serde_json::to_value(task).expect("serialize valid task baseline");
wire[field] = serde_json::from_str(ONE_OVER_U64)
.expect("retain the oversized mathematical JSON integer");
let error = serde_json::from_value::<FinalTask>(wire)
.expect_err("unrepresentable duration must fail typed admission");
assert_eq!(error.classify(), serde_json::error::Category::Data);
assert!(
store
.get_task(&task_id)
.expect("rejected task lookup remains readable")
.is_none(),
"the rejected {field} duration cannot reach retained task state"
);
assert!(store.latest_notification(&task_id).is_none());
assert_eq!(store.task_count(), 0);
}
}
#[test]
fn task_03_in_memory_store_positive_ttl_one_millisecond_before_deadline_preserves_state() {
const TTL_MS: u64 = 60_000;
let (store, now) = in_memory_store_with_test_clock(1);
let first = final_working_task_with_ttl("task-positive-ttl-first", TTL_MS);
let first_id = first.base().task_id.clone();
store
.create_task(first.clone(), final_task_notification(&first))
.expect("first task fits the bounded store");
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(TTL_MS - 1))
.expect("pre-deadline duration fits the monotonic test clock");
drop(clock);
let second = final_working_task_without_ttl("task-positive-ttl-second");
assert!(
store
.create_task(second.clone(), final_task_notification(&second))
.is_err(),
"only advancing the clock by one fewer millisecond preserves the first task"
);
assert_eq!(store.task_count(), 1);
assert!(
store
.get_task(&first_id)
.expect("pre-deadline task lookup is readable")
.is_some()
);
}
#[test]
fn task_03_in_memory_store_absent_ttl_has_no_automatic_expiry() {
const ELAPSED_MS: u64 = 60_000;
let (store, now) = in_memory_store_with_test_clock(1);
let first = final_working_task_without_ttl("task-absent-ttl-first");
let first_id = first.base().task_id.clone();
store
.create_task(first.clone(), final_task_notification(&first))
.expect("first task without a TTL fits the bounded store");
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(ELAPSED_MS))
.expect("test clock can advance without an automatic task expiry");
drop(clock);
let second = final_working_task_without_ttl("task-absent-ttl-second");
assert!(
store
.create_task(second.clone(), final_task_notification(&second))
.is_err(),
"an omitted TTL leaves the first task retained and capacity unavailable"
);
assert_eq!(store.task_count(), 1);
assert!(
store
.get_task(&first_id)
.expect("task without a TTL remains readable")
.is_some()
);
}
#[test]
fn task_03_final_runtime_emits_null_ttl_and_retains_without_automatic_expiry() {
const ELAPSED_MS: u64 = 60_000;
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_unlimited_ttl(&AllowUnlimitedFinalTaskRetention, None)
.expect("explicit authority admits null TTL retention"),
Arc::new(|_| {}),
);
let service_runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install caller-owned service runner for task creation");
let service_cx = Cx::for_testing();
let _running_service = enter_task_service_runner(service_runner, &service_cx);
let created = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("authorized null-TTL task is durably created");
let task_id = created.task.base().task_id.clone();
assert!(created.task.base().ttl_ms.is_none());
assert_eq!(runtime.config.ttl_ms(), None);
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(ELAPSED_MS))
.expect("test clock can advance past an unlimited task lifetime");
drop(clock);
assert!(
runtime.get_task(&task_id).is_ok(),
"only null TTL differs from the finite deadline case, so it remains retained"
);
assert_eq!(store.task_count(), 1);
}
#[test]
fn task_03_final_runtime_rejects_null_ttl_without_retention_authority() {
assert!(
FinalTaskRuntimeConfig::with_ttl(None, None).is_err(),
"only omitting the explicit retention authority rejects unlimited task retention"
);
}
#[test]
fn task_03_final_runtime_finite_ttl_reclaims_at_the_same_deadline() {
const TTL_MS: u64 = 60_000;
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_ttl(Some(TTL_MS), None)
.expect("positive TTL is a valid Task retention value"),
Arc::new(|_| {}),
);
let service_runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install caller-owned service runner for task creation");
let service_cx = Cx::for_testing();
let _running_service = enter_task_service_runner(service_runner, &service_cx);
let created = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("finite-TTL task is durably created");
let task_id = created.task.base().task_id.clone();
assert!(created.task.base().ttl_ms.is_some());
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(TTL_MS))
.expect("test clock reaches the finite task deadline");
drop(clock);
assert!(
runtime.get_task(&task_id).is_err(),
"changing only null TTL to a positive TTL permits reclamation at its deadline"
);
assert_eq!(store.task_count(), 0);
}
#[test]
fn task_02_final_expired_handoff_claim_releases_work_for_successor_recovery() {
let (store, now) = in_memory_store_with_test_clock(3);
let expired = final_working_task_with_ttl("task-expired-handoff-claim", 1_000);
let expired_id = expired.base().task_id.clone();
let stale_election = final_working_task_with_ttl("task-stale-handoff-election", 1_000);
let stale_election_id = stale_election.base().task_id.clone();
let successor = final_working_task_with_ttl("task-successor-handoff-claim", 10_000);
let successor_id = successor.base().task_id.clone();
let work_descriptor = final_test_work_descriptor();
store
.create_task_with_work(
expired.clone(),
final_task_notification(&expired),
work_descriptor.clone(),
)
.expect("expired candidate creates with a durable work descriptor");
store
.create_task_with_work(
stale_election.clone(),
final_task_notification(&stale_election),
work_descriptor.clone(),
)
.expect("stale election candidate creates with a durable work descriptor");
store
.create_task_with_work(
successor.clone(),
final_task_notification(&successor),
work_descriptor.clone(),
)
.expect("successor candidate creates with a durable work descriptor");
let expired_snapshot = store
.get_task_snapshot(&expired_id)
.expect("expired candidate snapshot is readable before the boundary")
.expect("expired candidate is retained before the boundary");
let stale_election_snapshot = store
.get_task_snapshot(&stale_election_id)
.expect("stale election snapshot is readable before the boundary")
.expect("stale election candidate is retained before the boundary");
let stale_election_claim = store
.take_initial_work_handoff_for_owner_if_current(&stale_election_snapshot, "old-owner")
.expect("stale election claim is readable before the boundary")
.expect("stale election candidate is claimable before the boundary");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) += StdDuration::from_millis(1_001);
assert!(
store
.take_initial_work_handoff_for_owner_if_current(&expired_snapshot, "old-owner")
.expect("expired claim boundary is readable")
.is_none(),
"retention expiry must prevent an old snapshot from acquiring ownership"
);
assert!(
store
.get_task(&expired_id)
.expect("expired candidate cleanup is readable")
.is_none(),
"expired work must be reclaimed before successor recovery"
);
assert!(
store
.begin_handoff_dispatch_for_owner_if_current(
&stale_election_id,
stale_election_claim.generation,
"old-owner",
)
.expect("expired dispatch election boundary is readable")
.is_none(),
"retention expiry must prevent an old claim from electing dispatch"
);
let successor_snapshot = store
.next_initial_work_snapshot()
.expect("successor recovery scan is readable")
.expect("successor remains eligible after the expired candidate is reclaimed");
assert_eq!(successor_snapshot.task().base().task_id, successor_id);
let successor_claim = store
.take_initial_work_handoff_for_owner_if_current(&successor_snapshot, "new-owner")
.expect("successor owner claim is readable")
.expect("successor claim remains eligible");
assert_eq!(successor_claim.task_id, successor_id);
assert!(
store
.begin_handoff_dispatch_for_owner_if_current(
&successor_id,
successor_claim.generation,
"new-owner",
)
.expect("successor dispatch election is readable")
.is_some(),
"successor must be electable through the real store after expiry cleanup"
);
}
#[test]
fn task_02_final_unexpired_handoff_claim_retains_live_owner() {
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_with_ttl("task-unexpired-handoff-claim", 1_000);
let task_id = task.base().task_id.clone();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.expect("unexpired candidate creates with a durable work descriptor");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("unexpired candidate snapshot is readable")
.expect("unexpired candidate is retained");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) += StdDuration::from_millis(999);
let claim = store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "live-owner")
.expect("unexpired owner claim is readable")
.expect("live owner retains its claim before the retention boundary");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(&task_id, claim.generation, "live-owner")
.expect("unexpired dispatch election is readable")
.expect("live owner retains dispatch election before expiry");
assert!(
store
.renew_handoff_dispatch_if_current(
&task_id,
claim.generation,
"live-owner",
dispatch_fence,
)
.expect("live owner heartbeat is readable")
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let lease = state
.handoff_leases
.get(&task_id)
.expect("live owner lease remains retained");
assert_eq!(lease.owner_id, "live-owner");
assert!(lease.dispatch_elected);
assert_eq!(lease.dispatch_fence, Some(dispatch_fence));
assert!(matches!(
state.tasks.get(&task_id),
Some(FinalTask::Working(_))
));
}
#[test]
fn task_02_final_initial_restoration_rejects_changed_payload_without_releasing_owner() {
for elected in [false, true] {
let (store, _now) = in_memory_store_with_test_clock(1);
let task = final_working_task_with_ttl("task-initial-restore-owner", 60_000);
let task_id = task.base().task_id.clone();
let descriptor = final_test_work_descriptor();
store
.create_task_with_authenticated_work(
task.clone(),
final_task_notification(&task),
descriptor.clone(),
Sha256Digest::from_bytes([7; 32]),
)
.expect("initial task retains its owner and operation");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial snapshot is readable")
.expect("initial task is retained");
store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "original-owner")
.expect("initial claim is readable")
.expect("original owner claims initial work");
let fence = elected.then(|| {
store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
snapshot.generation(),
"original-owner",
)
.expect("initial election is readable")
.expect("original owner elects dispatch")
});
let before = final_task_restoration_snapshot(&store, &task_id);
let altered = FinalTaskWorkDescriptor::new(serde_json::json!({
"handler": "tasks-test",
"payload": {"fixture": "substituted-task"}
}))
.expect("altered descriptor is structurally valid");
assert_ne!(altered, descriptor);
for (generation, candidate) in [
(snapshot.generation(), altered),
(snapshot.generation() + 1, descriptor.clone()),
] {
assert!(
!store
.restore_initial_work_for_owner_if_current(
&task_id,
generation,
"original-owner",
fence,
candidate,
)
.expect("mismatched initial restoration returns a refusal")
);
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
assert!(
store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "second-owner")
.expect("competing initial claim is readable")
.is_none(),
"a rejected restore cannot expose still-owned work to another runner"
);
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
}
assert!(
store
.restore_initial_work_for_owner_if_current(
&task_id,
snapshot.generation(),
"original-owner",
fence,
descriptor.clone(),
)
.expect("the matching initial restoration remains available")
);
let successor = store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "second-owner")
.expect("successor initial claim is readable")
.expect("matching restoration releases work to a successor");
assert_eq!(successor.work_descriptor, descriptor);
let successor_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
snapshot.generation(),
"second-owner",
)
.expect("successor election is readable")
.expect("successor elects dispatch");
if let Some(fence) = fence {
assert_ne!(successor_fence, fence);
}
let completed = FinalTask::Completed {
base: transition_terminal_final_task_base(
snapshot.task().base().clone(),
FinalTaskStatus::Completed,
None,
)
.expect("successor terminal base is valid"),
result: serde_json::from_value(serde_json::json!({"content": []}))
.expect("terminal result is valid"),
};
assert!(
store
.replace_task_and_clear_input_for_handoff_if_current(
&snapshot,
"second-owner",
successor_fence,
false,
completed.clone(),
final_task_notification(&completed),
)
.expect("successor completes under its own fence")
.is_some()
);
let terminal = final_task_restoration_snapshot(&store, &task_id);
assert!(
!store
.restore_initial_work_for_owner_if_current(
&task_id,
snapshot.generation(),
"second-owner",
Some(successor_fence),
descriptor,
)
.expect("terminal initial restoration returns a refusal")
);
assert_eq!(final_task_restoration_snapshot(&store, &task_id), terminal);
}
}
#[test]
fn task_02_final_input_restoration_rejects_changed_payload_without_releasing_owner() {
for elected in [false, true] {
let (store, _now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let inputs: FinalTaskInputResponses = serde_json::from_value(serde_json::json!({
"roots": {"roots": [{"uri": "file:///retained-input"}]}
}))
.expect("original accepted input is typed");
let task_id = create_accepted_final_input(&runtime, inputs.clone());
let snapshot = store
.get_task_snapshot(&task_id)
.expect("resumed snapshot is readable")
.expect("resumed task is retained");
store
.take_input_handoff_for_owner_if_current(&snapshot, "original-owner")
.expect("input claim is readable")
.expect("original owner claims accepted input");
let fence = elected.then(|| {
store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
snapshot.generation(),
"original-owner",
)
.expect("resumed election is readable")
.expect("original owner elects resumed dispatch")
});
let before = final_task_restoration_snapshot(&store, &task_id);
let altered: FinalTaskInputResponses = serde_json::from_value(serde_json::json!({
"roots": {"roots": [{"uri": "file:///substituted-input"}]}
}))
.expect("altered accepted input is also typed");
for (generation, candidate) in [
(snapshot.generation(), altered),
(snapshot.generation() + 1, inputs.clone()),
] {
assert!(
!store
.restore_input_for_owner_if_current(
&task_id,
generation,
"original-owner",
fence,
candidate,
)
.expect("mismatched input restoration returns a refusal")
);
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
assert!(
store
.take_input_handoff_for_owner_if_current(&snapshot, "second-owner")
.expect("competing input claim is readable")
.is_none(),
"a rejected restore cannot expose still-owned inputs to another runner"
);
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
}
assert!(
store
.restore_input_for_owner_if_current(
&task_id,
snapshot.generation(),
"original-owner",
fence,
inputs.clone(),
)
.expect("the matching input restoration remains available")
);
let successor = store
.take_input_handoff_for_owner_if_current(&snapshot, "second-owner")
.expect("successor input claim is readable")
.expect("matching restoration releases inputs to a successor");
assert_eq!(successor.input_responses, inputs);
let successor_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
snapshot.generation(),
"second-owner",
)
.expect("successor resumed election is readable")
.expect("successor elects resumed dispatch");
if let Some(fence) = fence {
assert_ne!(successor_fence, fence);
}
let completed = FinalTask::Completed {
base: transition_terminal_final_task_base(
snapshot.task().base().clone(),
FinalTaskStatus::Completed,
None,
)
.expect("successor terminal base is valid"),
result: serde_json::from_value(serde_json::json!({"content": []}))
.expect("terminal result is valid"),
};
assert!(
store
.replace_task_and_clear_input_for_handoff_if_current(
&snapshot,
"second-owner",
successor_fence,
false,
completed.clone(),
final_task_notification(&completed),
)
.expect("successor completes with its resumed fence")
.is_some()
);
let terminal = final_task_restoration_snapshot(&store, &task_id);
assert!(
!store
.restore_input_for_owner_if_current(
&task_id,
snapshot.generation(),
"second-owner",
Some(successor_fence),
inputs,
)
.expect("terminal input restoration returns a refusal")
);
assert_eq!(final_task_restoration_snapshot(&store, &task_id), terminal);
}
}
#[test]
fn task_02_final_expired_dispatch_lease_fences_initial_owner_renew_finish_restore_and_recovers()
{
let work_descriptor = final_test_work_descriptor();
{
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_with_ttl("task-expired-dispatch-initial-replace", 60_000);
let task_id = task.base().task_id.clone();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial snapshot is readable")
.expect("initial task is retained");
let claim = store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "stale-owner")
.expect("initial owner claim is readable")
.expect("initial owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
)
.expect("initial dispatch election is readable")
.expect("initial owner wins dispatch election");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
let replacement = FinalTask::Completed {
base: transition_terminal_final_task_base(
snapshot.task().base().clone(),
FinalTaskStatus::Completed,
Some("stale owner replacement must be fenced".to_owned()),
)
.expect("construct terminal replacement for the stale owner"),
result,
};
assert!(
store
.replace_task_and_clear_input_for_handoff_if_current(
&snapshot,
"stale-owner",
dispatch_fence,
false,
replacement.clone(),
final_task_notification(&replacement),
)
.expect("expired replacement refusal is readable")
.is_none()
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(state.initial_work.get(&task_id), Some(&work_descriptor));
assert!(matches!(
state.tasks.get(&task_id),
Some(FinalTask::Working(_))
));
drop(state);
let successor = store
.next_initial_work_snapshot()
.expect("initial recovery scan is readable")
.expect("expired replacement leaves initial work recoverable");
assert_eq!(
store
.take_initial_work_handoff_for_owner_if_current(&successor, "new-owner")
.expect("successor initial claim is readable")
.expect("successor can recover retained initial work")
.work_descriptor,
work_descriptor
);
}
{
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_with_ttl("task-expired-dispatch-initial-renew", 60_000);
let task_id = task.base().task_id.clone();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial snapshot is readable")
.expect("initial task is retained");
let claim = store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "stale-owner")
.expect("initial owner claim is readable")
.expect("initial owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
)
.expect("initial dispatch election is readable")
.expect("initial owner wins dispatch election");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
assert!(
!store
.renew_handoff_dispatch_if_current(
&task_id,
claim.generation,
"stale-owner",
dispatch_fence,
)
.expect("expired renewal refusal is readable")
);
let successor = store
.next_initial_work_snapshot()
.expect("initial recovery scan is readable")
.expect("expired dispatch owner leaves retained work recoverable");
assert_ne!(successor.generation(), claim.generation);
assert_eq!(
store
.take_initial_work_handoff_for_owner_if_current(&successor, "new-owner")
.expect("successor initial claim is readable")
.expect("successor can recover retained initial work")
.work_descriptor,
work_descriptor
);
}
{
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_with_ttl("task-expired-dispatch-initial-finish", 60_000);
let task_id = task.base().task_id.clone();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial snapshot is readable")
.expect("initial task is retained");
let claim = store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "stale-owner")
.expect("initial owner claim is readable")
.expect("initial owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
)
.expect("initial dispatch election is readable")
.expect("initial owner wins dispatch election");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
assert!(
!store
.finish_handoff_dispatch_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
dispatch_fence,
)
.expect("expired completion refusal is readable")
);
let successor = store
.next_initial_work_snapshot()
.expect("initial recovery scan is readable")
.expect("expired completion leaves retained work recoverable");
assert_eq!(
store
.take_initial_work_handoff_for_owner_if_current(&successor, "new-owner")
.expect("successor initial claim is readable")
.expect("successor can recover retained initial work")
.work_descriptor,
work_descriptor
);
}
{
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_with_ttl("task-expired-dispatch-initial-restore", 60_000);
let task_id = task.base().task_id.clone();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial snapshot is readable")
.expect("initial task is retained");
let claim = store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "stale-owner")
.expect("initial owner claim is readable")
.expect("initial owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
)
.expect("initial dispatch election is readable")
.expect("initial owner wins dispatch election");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
assert!(
!store
.restore_initial_work_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
Some(dispatch_fence),
work_descriptor.clone(),
)
.expect("expired restoration refusal is readable")
);
let successor = store
.next_initial_work_snapshot()
.expect("initial recovery scan is readable")
.expect("expired restoration leaves retained work recoverable");
assert_eq!(
store
.take_initial_work_handoff_for_owner_if_current(&successor, "new-owner")
.expect("successor initial claim is readable")
.expect("successor can recover retained initial work")
.work_descriptor,
work_descriptor
);
}
}
#[test]
fn task_02_final_unexpired_dispatch_lease_retains_initial_owner() {
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_with_ttl("task-unexpired-dispatch-initial", 60_000);
let task_id = task.base().task_id.clone();
let work_descriptor = final_test_work_descriptor();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial snapshot is readable")
.expect("initial task is retained");
let claim = store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "live-owner")
.expect("initial owner claim is readable")
.expect("initial owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(&task_id, claim.generation, "live-owner")
.expect("initial dispatch election is readable")
.expect("initial owner wins dispatch election");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE
.checked_sub(StdDuration::from_millis(1))
.expect("handoff lease exceeds one millisecond");
assert!(
store
.renew_handoff_dispatch_if_current(
&task_id,
claim.generation,
"live-owner",
dispatch_fence,
)
.expect("live renewal is readable")
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let lease = state
.handoff_leases
.get(&task_id)
.expect("live dispatch lease remains retained");
assert_eq!(lease.owner_id, "live-owner");
assert_eq!(lease.dispatch_fence, Some(dispatch_fence));
assert_eq!(state.initial_work.get(&task_id), Some(&work_descriptor));
assert!(matches!(
state.tasks.get(&task_id),
Some(FinalTask::Working(_))
));
}
#[test]
fn task_02_final_expired_dispatch_lease_fences_resumed_owner_renew_finish_restore_and_recovers()
{
let input_responses: FinalTaskInputResponses = serde_json::from_value(
serde_json::json!({"roots": {"roots": [{"uri": "file:///expired-dispatch"}]}}),
)
.expect("typed retained roots response");
{
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let snapshot = store
.get_task_snapshot(&task_id)
.expect("accepted-input snapshot is readable")
.expect("accepted-input task is retained");
let claim = store
.take_input_handoff_for_owner_if_current(&snapshot, "stale-owner")
.expect("accepted-input owner claim is readable")
.expect("accepted-input owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
)
.expect("resumed dispatch election is readable")
.expect("resumed owner wins dispatch election");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
let replacement = FinalTask::Completed {
base: transition_terminal_final_task_base(
snapshot.task().base().clone(),
FinalTaskStatus::Completed,
Some("stale owner replacement must be fenced".to_owned()),
)
.expect("construct terminal replacement for the stale owner"),
result,
};
assert!(
store
.replace_task_and_clear_input_for_handoff_if_current(
&snapshot,
"stale-owner",
dispatch_fence,
false,
replacement.clone(),
final_task_notification(&replacement),
)
.expect("expired replacement refusal is readable")
.is_none()
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(state.accepted_inputs.get(&task_id), Some(&input_responses));
assert!(matches!(
state.tasks.get(&task_id),
Some(FinalTask::Working(_))
));
drop(state);
let successor = store
.next_accepted_input_snapshot()
.expect("accepted-input recovery scan is readable")
.expect("expired replacement leaves accepted input recoverable");
assert_eq!(
store
.take_input_handoff_for_owner_if_current(&successor, "new-owner")
.expect("successor accepted-input claim is readable")
.expect("successor can recover accepted input")
.input_responses,
input_responses
);
}
{
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let snapshot = store
.get_task_snapshot(&task_id)
.expect("accepted-input snapshot is readable")
.expect("accepted-input task is retained");
let claim = store
.take_input_handoff_for_owner_if_current(&snapshot, "stale-owner")
.expect("accepted-input owner claim is readable")
.expect("accepted-input owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
)
.expect("resumed dispatch election is readable")
.expect("resumed owner wins dispatch election");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
assert!(
!store
.renew_handoff_dispatch_if_current(
&task_id,
claim.generation,
"stale-owner",
dispatch_fence,
)
.expect("expired resumed renewal refusal is readable")
);
let successor = store
.next_accepted_input_snapshot()
.expect("accepted-input recovery scan is readable")
.expect("expired resumed lease leaves accepted input recoverable");
assert_ne!(successor.generation(), claim.generation);
assert_eq!(
store
.take_input_handoff_for_owner_if_current(&successor, "new-owner")
.expect("successor accepted-input claim is readable")
.expect("successor can recover accepted input")
.input_responses,
input_responses
);
}
{
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let snapshot = store
.get_task_snapshot(&task_id)
.expect("accepted-input snapshot is readable")
.expect("accepted-input task is retained");
let claim = store
.take_input_handoff_for_owner_if_current(&snapshot, "stale-owner")
.expect("accepted-input owner claim is readable")
.expect("accepted-input owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
)
.expect("resumed dispatch election is readable")
.expect("resumed owner wins dispatch election");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
assert!(
!store
.finish_handoff_dispatch_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
dispatch_fence,
)
.expect("expired resumed completion refusal is readable")
);
let successor = store
.next_accepted_input_snapshot()
.expect("accepted-input recovery scan is readable")
.expect("expired completion leaves accepted input recoverable");
assert_eq!(
store
.take_input_handoff_for_owner_if_current(&successor, "new-owner")
.expect("successor accepted-input claim is readable")
.expect("successor can recover accepted input")
.input_responses,
input_responses
);
}
{
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let snapshot = store
.get_task_snapshot(&task_id)
.expect("accepted-input snapshot is readable")
.expect("accepted-input task is retained");
let claim = store
.take_input_handoff_for_owner_if_current(&snapshot, "stale-owner")
.expect("accepted-input owner claim is readable")
.expect("accepted-input owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
)
.expect("resumed dispatch election is readable")
.expect("resumed owner wins dispatch election");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
assert!(
!store
.restore_input_for_owner_if_current(
&task_id,
claim.generation,
"stale-owner",
Some(dispatch_fence),
input_responses.clone(),
)
.expect("expired resumed restoration refusal is readable")
);
let successor = store
.next_accepted_input_snapshot()
.expect("accepted-input recovery scan is readable")
.expect("expired restoration leaves accepted input recoverable");
assert_eq!(
store
.take_input_handoff_for_owner_if_current(&successor, "new-owner")
.expect("successor accepted-input claim is readable")
.expect("successor can recover accepted input")
.input_responses,
input_responses
);
}
}
#[test]
fn task_02_final_unexpired_dispatch_lease_retains_resumed_owner() {
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let snapshot = store
.get_task_snapshot(&task_id)
.expect("accepted-input snapshot is readable")
.expect("accepted-input task is retained");
let claim = store
.take_input_handoff_for_owner_if_current(&snapshot, "live-owner")
.expect("accepted-input owner claim is readable")
.expect("accepted-input owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(&task_id, claim.generation, "live-owner")
.expect("resumed dispatch election is readable")
.expect("resumed owner wins dispatch election");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE
.checked_sub(StdDuration::from_millis(1))
.expect("handoff lease exceeds one millisecond");
assert!(
store
.renew_handoff_dispatch_if_current(
&task_id,
claim.generation,
"live-owner",
dispatch_fence,
)
.expect("live resumed renewal is readable")
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let lease = state
.handoff_leases
.get(&task_id)
.expect("live resumed dispatch lease remains retained");
assert_eq!(lease.owner_id, "live-owner");
assert_eq!(lease.dispatch_fence, Some(dispatch_fence));
assert_eq!(state.accepted_inputs.get(&task_id), Some(&input_responses));
assert!(matches!(
state.tasks.get(&task_id),
Some(FinalTask::Working(_))
));
}
#[test]
fn task_02_final_expired_resumed_input_claim_releases_work_for_successor_recovery() {
let (store, now) = in_memory_store_with_test_clock(2);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(1_000, None).expect("finite retention policy is valid"),
Arc::new(|_| {}),
);
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let expired_id = create_accepted_final_input(&runtime, input_responses.clone());
let expired_snapshot = store
.get_task_snapshot(&expired_id)
.expect("expired candidate snapshot is readable before the boundary")
.expect("expired candidate is retained before the boundary");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) += StdDuration::from_millis(1_000);
assert!(
store
.take_input_handoff_for_owner_if_current(&expired_snapshot, "old-owner")
.expect("expired resumed claim boundary is readable")
.is_none()
);
assert!(
store
.get_task(&expired_id)
.expect("expired resumed task cleanup is readable")
.is_none()
);
let successor_id = create_accepted_final_input(&runtime, input_responses.clone());
let successor = store
.next_accepted_input_snapshot()
.expect("successor accepted-input scan is readable")
.expect("successor remains eligible after expired cleanup");
assert_eq!(successor.task().base().task_id, successor_id);
let claim = store
.take_input_handoff_for_owner_if_current(&successor, "new-owner")
.expect("successor resumed claim is readable")
.expect("successor resumed claim remains eligible");
assert_eq!(claim.input_responses, input_responses);
}
#[test]
fn task_02_final_unexpired_resumed_input_claim_retains_live_owner() {
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(1_000, None).expect("finite retention policy is valid"),
Arc::new(|_| {}),
);
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let snapshot = store
.get_task_snapshot(&task_id)
.expect("accepted-input snapshot is readable")
.expect("accepted-input task is retained");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) += StdDuration::from_millis(999);
let claim = store
.take_input_handoff_for_owner_if_current(&snapshot, "live-owner")
.expect("unexpired resumed claim is readable")
.expect("live owner retains accepted-input claim before the boundary");
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(state.accepted_inputs.get(&task_id), Some(&input_responses));
assert_eq!(state.handoff_leases[&task_id].owner_id, "live-owner");
assert_eq!(state.handoff_leases[&task_id].generation, claim.generation);
assert!(matches!(
state.tasks.get(&task_id),
Some(FinalTask::Working(_))
));
}
#[test]
fn task_02_final_clock_sampling_is_linearized_with_fenced_lease_transition() {
let now = Arc::new(Mutex::new(Instant::now()));
let clock_now = Arc::clone(&now);
let armed = Arc::new(AtomicBool::new(false));
let clock_armed = Arc::clone(&armed);
let (sampled_sender, sampled_receiver) = std::sync::mpsc::sync_channel(1);
let (release_sender, release_receiver) = std::sync::mpsc::sync_channel(1);
let release_receiver = Arc::new(Mutex::new(release_receiver));
let clock_release_receiver = Arc::clone(&release_receiver);
let clock: Arc<dyn Fn() -> Instant + Send + Sync> = Arc::new(move || {
let sampled_now = *clock_now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if clock_armed.swap(false, AtomicOrdering::SeqCst) {
sampled_sender
.send(sampled_now)
.expect("clock callback consumer remains available");
clock_release_receiver
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.recv_timeout(StdDuration::from_secs(1))
.expect("clock callback release is bounded");
}
sampled_now
});
let store = Arc::new(
InMemoryFinalTaskStore::with_clock(1, clock)
.expect("positive bounded store capacity is valid"),
);
let task = final_working_task_without_ttl("task-clock-linearization");
let task_id = task.base().task_id.clone();
let work_descriptor = final_test_work_descriptor();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial snapshot is readable")
.expect("initial task is retained");
let claim = store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "clock-owner")
.expect("initial owner claim is readable")
.expect("initial owner claim succeeds");
let dispatch_fence = store
.begin_handoff_dispatch_for_owner_if_current(&task_id, claim.generation, "clock-owner")
.expect("initial dispatch election is readable")
.expect("initial owner wins dispatch election");
{
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock += IN_MEMORY_FINAL_TASK_HANDOFF_LEASE
.checked_sub(StdDuration::from_millis(1))
.expect("handoff lease exceeds one millisecond");
}
armed.store(true, AtomicOrdering::SeqCst);
let worker_store = Arc::clone(&store);
let worker_task_id = task_id.clone();
let worker = thread::spawn(move || {
FinalTaskStore::renew_handoff_dispatch_if_current(
&*worker_store,
&worker_task_id,
claim.generation,
"clock-owner",
dispatch_fence,
)
});
let sampled_at = sampled_receiver
.recv_timeout(StdDuration::from_secs(1))
.expect("worker reaches the armed clock callback");
let parent_won_state_lock = match store.state.try_lock() {
Ok(state) => {
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = sampled_at
.checked_add(StdDuration::from_millis(1))
.expect("test clock reaches the exact dispatch boundary");
drop(state);
true
}
Err(std::sync::TryLockError::WouldBlock) => false,
Err(std::sync::TryLockError::Poisoned(_)) => {
panic!("state mutex is not poisoned during clock ordering probe")
}
};
release_sender
.send(())
.expect("worker clock callback remains blocked until released");
let renewed = worker
.join()
.expect("bounded clock ordering worker exits without panic")
.expect("clock ordering renewal returns a decision");
if parent_won_state_lock {
assert!(
!renewed,
"expired pre-lock clock sample must not authorize renewal"
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(state.initial_work.get(&task_id), Some(&work_descriptor));
} else {
assert!(
renewed,
"live owner remains valid when it serializes clock sampling"
);
}
}
fn execution_test_store(
total_ms: u64,
attempt_ms: u64,
max_attempts: u32,
) -> (Arc<InMemoryFinalTaskStore>, Arc<Mutex<Instant>>) {
let clock = Arc::new(Mutex::new(Instant::now()));
let clock_reader = Arc::clone(&clock);
let limits = FinalTaskExecutionLimits::new(
StdDuration::from_millis(total_ms),
StdDuration::from_millis(attempt_ms),
max_attempts,
)
.unwrap();
let store = InMemoryFinalTaskStore::with_clock_payload_and_execution_limits(
4,
InMemoryFinalTaskStore::DEFAULT_MAX_PAYLOAD_BYTES,
limits,
Arc::new(move || *clock_reader.lock().unwrap()),
)
.unwrap();
(Arc::new(store), clock)
}
#[derive(Default)]
struct ExecutionPollingSupervisor {
require_initial_input: bool,
polls: AtomicUsize,
dropped: Arc<AtomicBool>,
budgets: Mutex<Vec<FinalTaskExecutionBudget>>,
}
impl ApplicationTaskSupervisor for ExecutionPollingSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
self.budgets
.lock()
.unwrap()
.push(handoff.execution_budget()?);
let finishes = match &handoff {
FinalTaskSupervisorHandoff::Initial(initial) => initial.work_descriptor(),
FinalTaskSupervisorHandoff::Resumed(accepted) => accepted.work_descriptor(),
}
.as_value()
.get("finish")
.and_then(serde_json::Value::as_bool)
== Some(true);
if finishes {
let result =
serde_json::from_value(serde_json::json!({"content": []})).unwrap();
match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => {
initial.complete_task(result, None)?;
}
FinalTaskSupervisorHandoff::Resumed(accepted) => {
accepted.complete_task(result, None)?;
}
}
return Ok(());
}
if self.require_initial_input
&& let FinalTaskSupervisorHandoff::Initial(initial) = &handoff
{
initial.require_input(final_roots_request(), None)?;
return Ok(());
}
let _lifetime = RetentionExpiryDropFlag(Arc::clone(&self.dropped));
let _handoff = handoff;
std::future::poll_fn(|_| {
self.polls.fetch_add(1, AtomicOrdering::SeqCst);
std::task::Poll::Pending::<McpResult<()>>
})
.await
})
}
}
fn check_execution_before_application_repoll(expired: bool, total: bool) {
for resumed in [false, true] {
let (store, clock) = execution_test_store(
if total { 100 } else { 1_000 },
if total { 200 } else { 100 },
8,
);
let runtime = final_task_runtime(store.clone(), Arc::new(AtomicBool::new(false)));
let supervisor = Arc::new(ExecutionPollingSupervisor {
require_initial_input: resumed,
..ExecutionPollingSupervisor::default()
});
let mut runner = runtime.install_task_service(2, supervisor.clone()).unwrap();
let application = RuntimeBuilder::current_thread().build().unwrap();
application.block_on(async {
let cx = Cx::current().unwrap();
let service_budget = cx.budget();
let mut running = Box::pin(runner.run_service(&cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(running.as_mut().poll(&mut context).is_pending());
let created = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.unwrap();
let task_id = created.task.base().task_id.clone();
assert!(running.as_mut().poll(&mut context).is_pending());
let first = supervisor.budgets.lock().unwrap()[0];
if resumed {
assert!(matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::InputRequired { .. })
));
*clock.lock().unwrap() += StdDuration::from_millis(10);
let inputs =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.unwrap();
runtime.update_task(&task_id, &inputs).unwrap();
assert!(running.as_mut().poll(&mut context).is_pending());
}
let budget = *supervisor.budgets.lock().unwrap().last().unwrap();
assert_eq!(budget.total_deadline(), first.total_deadline());
assert_eq!(budget.attempts(), if resumed { 2 } else { 1 });
assert_eq!(supervisor.polls.load(AtomicOrdering::SeqCst), 1);
let deadline = budget.attempt_deadline().unwrap();
*clock.lock().unwrap() = if expired {
deadline
} else {
deadline - StdDuration::from_millis(1)
};
assert!(
running.as_mut().poll(&mut context).is_pending(),
"expiry must not fail the service"
);
let task = store.get_task(&task_id).unwrap().unwrap();
if expired {
assert_eq!(supervisor.polls.load(AtomicOrdering::SeqCst), 1);
assert!(supervisor.dropped.load(AtomicOrdering::SeqCst));
let FinalTask::Failed { error, .. } = &task else {
panic!("expired execution must retain failure");
};
assert_eq!(
error.message,
if total {
FINAL_TASK_TOTAL_DEADLINE_EXCEEDED
} else {
FINAL_TASK_ATTEMPT_DEADLINE_EXCEEDED
}
);
assert!(store.state.lock().unwrap().handoff_leases.is_empty());
assert_eq!(
serde_json::to_value(&task).unwrap(),
serde_json::to_value(
store.latest_notification(&task_id).unwrap().params.task,
)
.unwrap()
);
} else {
assert_eq!(supervisor.polls.load(AtomicOrdering::SeqCst), 2);
assert!(!supervisor.dropped.load(AtomicOrdering::SeqCst));
assert!(matches!(task, FinalTask::Working(_)));
}
assert!(runtime.is_task_service_ready());
assert_eq!(cx.budget().deadline, service_budget.deadline);
assert_eq!(
store.task_count(),
1,
"execution expiry must preserve retained result"
);
assert_final_task_payload_accounting(&store);
if expired {
let sibling = runtime
.create_task_with_work(
FinalTaskWorkDescriptor::new(serde_json::json!({"finish": true}))
.unwrap(),
None,
)
.unwrap();
assert!(running.as_mut().poll(&mut context).is_pending());
assert!(
matches!(
store.get_task(&sibling.task.base().task_id).unwrap(),
Some(FinalTask::Completed { .. })
),
"a deadline failure must leave the service usable by healthy work"
);
}
drop(running);
});
}
}
#[test]
fn task_02_total_execution_deadline_stops_initial_and_resumed_work() {
check_execution_before_application_repoll(true, true);
}
#[test]
fn task_02_live_total_execution_budget_allows_initial_and_resumed_work() {
check_execution_before_application_repoll(false, true);
}
#[test]
fn task_02_attempt_execution_deadline_stops_initial_and_resumed_work() {
check_execution_before_application_repoll(true, false);
}
#[test]
fn task_02_live_attempt_execution_budget_allows_initial_and_resumed_work() {
check_execution_before_application_repoll(false, false);
}
#[test]
fn task_02_execution_limits_validate_defaults_and_hard_ceilings() {
let defaults = FinalTaskExecutionLimits::default();
assert_eq!(defaults.total_runtime(), StdDuration::from_hours(24));
assert_eq!(defaults.attempt_runtime(), StdDuration::from_secs(3_600));
assert_eq!(defaults.max_attempts(), 8);
assert!(
FinalTaskExecutionLimits::new(
StdDuration::from_hours(720),
StdDuration::from_hours(24),
64,
)
.is_ok()
);
for (total, attempt, claims) in [
(0, 1, 1),
(1, 0, 1),
(1, 1, 0),
(30 * 86_400 + 1, 1, 1),
(1, 86_401, 1),
(1, 1, 65),
] {
assert!(
FinalTaskExecutionLimits::new(
StdDuration::from_secs(total),
StdDuration::from_secs(attempt),
claims,
)
.is_err()
);
}
}
fn check_input_wait_execution_recovery(expired: bool) {
let (store, clock) = execution_test_store(100, 40, 8);
let runtime = final_task_runtime(store.clone(), Arc::new(AtomicBool::new(false)));
let supervisor = Arc::new(ExecutionPollingSupervisor {
require_initial_input: true,
..ExecutionPollingSupervisor::default()
});
let mut runner = runtime.install_task_service(2, supervisor.clone()).unwrap();
RuntimeBuilder::current_thread()
.build()
.unwrap()
.block_on(async {
let cx = Cx::current().unwrap();
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
let mut running = Box::pin(runner.run_service(&cx));
assert!(running.as_mut().poll(&mut context).is_pending());
let task = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.unwrap()
.task;
let task_id = task.base().task_id.clone();
assert!(running.as_mut().poll(&mut context).is_pending());
let waiting = store.get_task_snapshot(&task_id).unwrap().unwrap();
assert!(matches!(waiting.task(), FinalTask::InputRequired { .. }));
let original = store
.task_execution_budget_if_current(&waiting)
.unwrap()
.unwrap();
assert_eq!(original.attempts(), 1);
assert!(
original.attempt_deadline().is_none(),
"waiting input has no active attempt"
);
drop(running);
drop(runner);
*clock.lock().unwrap() = original.total_deadline()
- if expired {
StdDuration::ZERO
} else {
StdDuration::from_millis(1)
};
let recovered = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(5_000, None).unwrap(),
Arc::new(|_| {}),
);
let mut successor = recovered
.install_task_service(2, supervisor.clone())
.unwrap();
let mut recovering = Box::pin(successor.run_service(&cx));
assert!(recovering.as_mut().poll(&mut context).is_pending());
let current = store.get_task_snapshot(&task_id).unwrap().unwrap();
let current_budget = store
.task_execution_budget_if_current(¤t)
.unwrap()
.unwrap();
assert_eq!(current_budget.total_deadline(), original.total_deadline());
assert_eq!(current.task().base().ttl_ms, task.base().ttl_ms);
let inputs =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}})).unwrap();
recovered.update_task(&task_id, &inputs).unwrap();
assert!(recovering.as_mut().poll(&mut context).is_pending());
if expired {
let FinalTask::Failed { error, .. } = current.task() else {
panic!("recovery must expire input wait");
};
assert_eq!(error.message, FINAL_TASK_TOTAL_DEADLINE_EXCEEDED);
assert_eq!(supervisor.budgets.lock().unwrap().len(), 1);
assert!(store.state.lock().unwrap().accepted_inputs.is_empty());
} else {
assert!(matches!(current.task(), FinalTask::InputRequired { .. }));
let budget = *supervisor.budgets.lock().unwrap().last().unwrap();
assert_eq!(budget.attempts(), 2);
assert_eq!(budget.attempt_deadline(), Some(original.total_deadline()));
assert_eq!(supervisor.polls.load(AtomicOrdering::SeqCst), 1);
}
assert!(recovered.is_task_service_ready());
drop(recovering);
});
}
#[test]
fn task_02_input_wait_total_deadline_survives_runtime_recovery() {
check_input_wait_execution_recovery(true);
}
#[test]
fn task_02_live_input_wait_resumes_with_original_total_deadline() {
check_input_wait_execution_recovery(false);
}
#[test]
fn task_02_renewal_and_restoration_preserve_execution_accounting() {
let (store, clock) = execution_test_store(1_000, 100, 2);
let task = final_working_task_with_ttl("task-execution-recovery", 60_000);
let task_id = task.base().task_id.clone();
let descriptor = final_test_work_descriptor();
let original = store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
descriptor.clone(),
)
.unwrap();
let created_budget = store
.task_execution_budget_if_current(&original)
.unwrap()
.unwrap();
let start = *clock.lock().unwrap();
store
.take_initial_work_handoff_for_owner_if_current(&original, "first")
.unwrap()
.unwrap();
let first = store
.task_execution_budget_if_current(&original)
.unwrap()
.unwrap();
let fence = store
.begin_handoff_dispatch_for_owner_if_current(&task_id, original.generation(), "first")
.unwrap()
.unwrap();
*clock.lock().unwrap() = start + StdDuration::from_millis(50);
assert!(
store
.renew_handoff_dispatch_if_current(&task_id, original.generation(), "first", fence)
.unwrap()
);
assert_eq!(
store
.task_execution_budget_if_current(&original)
.unwrap()
.unwrap(),
first
);
assert!(
store
.take_initial_work_handoff_for_owner_if_current(&original, "competitor")
.unwrap()
.is_none()
);
assert_eq!(
store
.task_execution_budget_if_current(&original)
.unwrap()
.unwrap()
.attempts(),
1
);
assert!(
store
.restore_initial_work_for_owner_if_current(
&task_id,
original.generation(),
"first",
Some(fence),
descriptor.clone(),
)
.unwrap()
);
let restored = store
.task_execution_budget_if_current(&original)
.unwrap()
.unwrap();
assert_eq!(restored.total_deadline(), created_budget.total_deadline());
assert_eq!(restored.attempts(), 1);
assert!(restored.attempt_deadline().is_none());
store
.take_initial_work_handoff_for_owner_if_current(&original, "second")
.unwrap()
.unwrap();
let second = store
.task_execution_budget_if_current(&original)
.unwrap()
.unwrap();
assert_eq!(second.attempts(), 2);
assert_eq!(second.total_deadline(), created_budget.total_deadline());
assert_eq!(
second.attempt_deadline(),
Some(start + StdDuration::from_millis(150))
);
assert!(
store
.expire_task_execution_if_current(&original)
.unwrap()
.is_none(),
"last allowed live attempt remains authorized"
);
assert!(
!store
.renew_handoff_dispatch_if_current(&task_id, original.generation(), "first", fence)
.unwrap()
);
assert!(
store
.restore_initial_work_for_owner_if_current(
&task_id,
original.generation(),
"second",
None,
descriptor,
)
.unwrap()
);
let terminal = store
.expire_task_execution_if_current(&original)
.unwrap()
.unwrap();
let FinalTask::Failed { error, .. } = terminal.task() else {
panic!("exhausted claims fail deterministically");
};
assert_eq!(error.message, FINAL_TASK_ATTEMPTS_EXHAUSTED);
assert!(
store
.take_initial_work_handoff_for_owner_if_current(&terminal, "third")
.unwrap()
.is_none()
);
assert_final_task_payload_accounting(&store);
}
#[test]
fn task_02_execution_failure_uses_reserved_capacity_and_retained_identity() {
let task = final_working_task_with_ttl("task-full-execution-expiry", 60_000);
let task_id = task.base().task_id.clone();
let payload = encoded_final_task_test_bytes(&task)
+ encoded_final_task_test_bytes(&final_task_notification(&task));
let clock = Arc::new(Mutex::new(Instant::now()));
let reader = clock.clone();
let store = InMemoryFinalTaskStore::with_clock_payload_and_execution_limits(
2,
payload + FINAL_TASK_CANCELLATION_RESERVE_BYTES,
FinalTaskExecutionLimits::new(
StdDuration::from_millis(100),
StdDuration::from_millis(100),
1,
)
.unwrap(),
Arc::new(move || *reader.lock().unwrap()),
)
.unwrap();
store
.create_task(task.clone(), final_task_notification(&task))
.unwrap();
let snapshot = store.get_task_snapshot(&task_id).unwrap().unwrap();
let before = final_task_restoration_snapshot(&store, &task_id);
let sibling = final_working_task_with_ttl("capacity-competitor", 60_000);
assert!(
store
.create_task(sibling.clone(), final_task_notification(&sibling))
.is_err()
);
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
assert!(
store
.expire_task_execution_if_current(&snapshot)
.unwrap()
.is_none()
);
*clock.lock().unwrap() += StdDuration::from_millis(100);
let mut forged_task = task.clone();
if let FinalTask::Working(base) = &mut forged_task {
base.ttl_ms = Some(final_task_duration(9_999).unwrap());
base.created_at = FinalTaskTimestamp::parse("2000-01-01T00:00:00Z").unwrap();
}
let forged = FinalTaskSnapshot::new(forged_task, snapshot.generation());
let expired = store
.expire_task_execution_if_current(&forged)
.unwrap()
.unwrap();
assert_eq!(expired.task().base().created_at, task.base().created_at);
assert_eq!(expired.task().base().ttl_ms, task.base().ttl_ms);
assert!(matches!(expired.task(), FinalTask::Failed { .. }));
assert!(store.retained_payload_bytes() <= store.max_payload_bytes());
assert_eq!(store.task_count(), 1);
assert_final_task_payload_accounting(&store);
}
#[test]
fn task_02_backend_fenced_completion_checks_exact_execution_deadline() {
for total in [false, true] {
for expired in [false, true] {
let (store, clock) = execution_test_store(
if total { 100 } else { 1_000 },
if total { 200 } else { 100 },
8,
);
let task = final_working_task_with_ttl("task-atomic-execution-boundary", 60_000);
let task_id = task.base().task_id.clone();
let expected = store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.unwrap();
store
.take_initial_work_handoff_for_owner_if_current(&expected, "owner")
.unwrap()
.unwrap();
let fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
expected.generation(),
"owner",
)
.unwrap()
.unwrap();
let budget = store
.task_execution_budget_if_current(&expected)
.unwrap()
.unwrap();
*clock.lock().unwrap() = budget.attempt_deadline().unwrap()
- if expired {
StdDuration::ZERO
} else {
StdDuration::from_millis(1)
};
let before = final_task_restoration_snapshot(&store, &task_id);
let completed = FinalTask::Completed {
base: transition_terminal_final_task_base(
task.base().clone(),
FinalTaskStatus::Completed,
None,
)
.unwrap(),
result: serde_json::from_value(serde_json::json!({"content": []})).unwrap(),
};
let write = store.replace_task_and_clear_input_for_handoff_if_current(
&expected,
"owner",
fence,
false,
completed.clone(),
final_task_notification(&completed),
);
if expired {
assert!(
write.is_err(),
"the atomic backend must refuse late completion without relying on a runner poll"
);
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
assert_eq!(
store
.task_execution_budget_if_current(&expected)
.unwrap()
.unwrap(),
budget
);
let terminal = store
.expire_task_execution_if_current(&expected)
.unwrap()
.unwrap();
let FinalTask::Failed { error, .. } = terminal.task() else {
panic!("deadline retirement must win");
};
assert_eq!(
error.message,
if total {
FINAL_TASK_TOTAL_DEADLINE_EXCEEDED
} else {
FINAL_TASK_ATTEMPT_DEADLINE_EXCEEDED
}
);
assert!(
!store
.finish_handoff_dispatch_for_owner_if_current(
&task_id,
expected.generation(),
"owner",
fence,
)
.unwrap()
);
} else {
assert!(write.unwrap().is_some());
assert!(matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::Completed { .. })
));
}
}
}
}
#[test]
fn task_02_lease_reclaim_spends_attempt_without_resetting_total_deadline() {
let (store, clock) = execution_test_store(120_000, 60_000, 8);
let task = final_working_task_with_ttl("task-budget-lease-reclaim", 180_000);
let task_id = task.base().task_id.clone();
let first = store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.unwrap();
store
.take_initial_work_handoff_for_owner_if_current(&first, "old")
.unwrap()
.unwrap();
let original = store
.task_execution_budget_if_current(&first)
.unwrap()
.unwrap();
*clock.lock().unwrap() += IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
let recovered = store.get_task_snapshot(&task_id).unwrap().unwrap();
assert!(recovered.generation() > first.generation());
let awaiting = store
.task_execution_budget_if_current(&recovered)
.unwrap()
.unwrap();
assert_eq!(awaiting.total_deadline(), original.total_deadline());
assert_eq!(awaiting.attempts(), 1);
assert!(awaiting.attempt_deadline().is_none());
store
.take_initial_work_handoff_for_owner_if_current(&recovered, "new")
.unwrap()
.unwrap();
let retry = store
.task_execution_budget_if_current(&recovered)
.unwrap()
.unwrap();
assert_eq!(retry.attempts(), 2);
assert_eq!(retry.total_deadline(), original.total_deadline());
assert_eq!(
retry.attempt_deadline(),
Some(*clock.lock().unwrap() + StdDuration::from_secs(60))
);
assert!(
store
.take_initial_work_handoff_for_owner_if_current(&first, "old")
.unwrap()
.is_none()
);
}
#[test]
fn task_02_queued_execution_expires_before_application_entry() {
for expired in [false, true] {
let (store, clock) = execution_test_store(100, 100, 8);
let runtime = final_task_runtime(store.clone(), Arc::new(AtomicBool::new(false)));
let supervisor = Arc::new(ExecutionPollingSupervisor::default());
let mut runner = runtime.install_task_service(1, supervisor.clone()).unwrap();
RuntimeBuilder::current_thread()
.build()
.unwrap()
.block_on(async {
let cx = Cx::current().unwrap();
let mut running = Box::pin(runner.run_service(&cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(running.as_mut().poll(&mut context).is_pending());
let task = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.unwrap()
.task;
*clock.lock().unwrap() +=
StdDuration::from_millis(if expired { 100 } else { 99 });
assert!(running.as_mut().poll(&mut context).is_pending());
let current = store.get_task(&task.base().task_id).unwrap().unwrap();
if expired {
assert!(matches!(current, FinalTask::Failed { .. }));
assert_eq!(supervisor.polls.load(AtomicOrdering::SeqCst), 0);
assert!(supervisor.budgets.lock().unwrap().is_empty());
} else {
assert!(matches!(current, FinalTask::Working(_)));
assert_eq!(supervisor.polls.load(AtomicOrdering::SeqCst), 1);
}
assert!(runtime.is_task_service_ready());
drop(running);
});
}
}
#[test]
fn task_02_execution_expiry_backlog_does_not_starve_live_initial_or_resumed_work() {
for resumed in [false, true] {
let expired_count = 2 * MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN + 2;
let clock = Arc::new(Mutex::new(Instant::now()));
let reader = clock.clone();
let store = Arc::new(
InMemoryFinalTaskStore::with_clock_payload_and_execution_limits(
expired_count + 1,
InMemoryFinalTaskStore::DEFAULT_MAX_PAYLOAD_BYTES,
FinalTaskExecutionLimits::new(
StdDuration::from_millis(100),
StdDuration::from_millis(100),
8,
)
.unwrap(),
Arc::new(move || *reader.lock().unwrap()),
)
.unwrap(),
);
let runtime = final_task_runtime(store.clone(), Arc::new(AtomicBool::new(false)));
let inputs =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}})).unwrap();
let descriptor =
FinalTaskWorkDescriptor::new(serde_json::json!({"finish": true})).unwrap();
for index in 0..=expired_count {
if index == expired_count {
*clock.lock().unwrap() += StdDuration::from_millis(100);
}
let task =
final_working_task_with_ttl(&format!("task-expiry-backlog-{index:04}"), 60_000);
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
descriptor.clone(),
)
.unwrap();
if resumed {
runtime
.require_input(&task.base().task_id, final_roots_request(), None)
.unwrap();
runtime.update_task(&task.base().task_id, &inputs).unwrap();
}
}
let live_id =
FinalTaskId::parse(format!("task-expiry-backlog-{expired_count:04}")).unwrap();
let supervisor = Arc::new(ExecutionPollingSupervisor::default());
let mut runner = runtime.install_task_service(4, supervisor.clone()).unwrap();
RuntimeBuilder::current_thread()
.build()
.unwrap()
.block_on(async {
let cx = Cx::current().unwrap();
let mut running = Box::pin(runner.run_service(&cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(
running.as_mut().poll(&mut context).is_pending(),
"large expired backlog must not exhaust work-recovery CAS retries"
);
assert!(runtime.is_task_service_ready());
assert!(matches!(
store.get_task(&live_id).unwrap(),
Some(FinalTask::Completed { .. })
));
assert_eq!(
supervisor.budgets.lock().unwrap().len(),
1,
"only the healthy record may enter application work"
);
assert!(
store.next_expired_execution_snapshot().unwrap().is_none(),
"bounded self-wake turns eventually retire the entire expired backlog"
);
assert_eq!(store.task_count(), expired_count + 1);
assert_final_task_payload_accounting(&store);
drop(running);
});
}
}
struct ExecutionCompletionSupervisor {
clock: Arc<Mutex<Instant>>,
complete_at_deadline: Option<bool>,
cleanup_polls: AtomicUsize,
rejected: AtomicBool,
dropped: Arc<AtomicBool>,
}
impl ApplicationTaskSupervisor for ExecutionCompletionSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
let _lifetime = RetentionExpiryDropFlag(self.dropped.clone());
let budget = handoff.execution_budget()?;
if let Some(expired) = self.complete_at_deadline {
*self.clock.lock().unwrap() = budget.attempt_deadline().unwrap()
- if expired {
StdDuration::ZERO
} else {
StdDuration::from_millis(1)
};
}
let result = serde_json::from_value(serde_json::json!({"content": []})).unwrap();
let committed = match &handoff {
FinalTaskSupervisorHandoff::Initial(initial) => {
initial.complete_task(result, None)
}
FinalTaskSupervisorHandoff::Resumed(accepted) => {
accepted.complete_task(result, None)
}
};
self.rejected
.store(committed.is_err(), AtomicOrdering::SeqCst);
committed?;
if self.complete_at_deadline.is_some() {
return Ok(());
}
std::future::poll_fn(|_| {
self.cleanup_polls.fetch_add(1, AtomicOrdering::SeqCst);
std::task::Poll::Pending::<McpResult<()>>
})
.await
})
}
}
fn check_execution_completion_boundary(expired: bool, cleanup: bool) {
let (store, clock) = execution_test_store(1_000, 100, 8);
let runtime = final_task_runtime(store.clone(), Arc::new(AtomicBool::new(false)));
let supervisor = Arc::new(ExecutionCompletionSupervisor {
clock: clock.clone(),
complete_at_deadline: (!cleanup).then_some(expired),
cleanup_polls: AtomicUsize::new(0),
rejected: AtomicBool::new(false),
dropped: Arc::new(AtomicBool::new(false)),
});
let mut runner = runtime.install_task_service(2, supervisor.clone()).unwrap();
RuntimeBuilder::current_thread()
.build()
.unwrap()
.block_on(async {
let cx = Cx::current().unwrap();
let mut running = Box::pin(runner.run_service(&cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(running.as_mut().poll(&mut context).is_pending());
let start = *clock.lock().unwrap();
let task = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.unwrap()
.task;
let task_id = task.base().task_id.clone();
assert!(running.as_mut().poll(&mut context).is_pending());
if cleanup {
assert_eq!(supervisor.cleanup_polls.load(AtomicOrdering::SeqCst), 1);
*clock.lock().unwrap() =
start + StdDuration::from_millis(if expired { 100 } else { 99 });
assert!(running.as_mut().poll(&mut context).is_pending());
assert_eq!(
supervisor.cleanup_polls.load(AtomicOrdering::SeqCst),
if expired { 1 } else { 2 }
);
assert_eq!(supervisor.dropped.load(AtomicOrdering::SeqCst), expired);
assert!(
matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::Completed { .. })
),
"bounded cleanup must preserve the already committed success"
);
} else {
assert_eq!(supervisor.rejected.load(AtomicOrdering::SeqCst), expired);
let current = store.get_task(&task_id).unwrap().unwrap();
if expired {
let FinalTask::Failed { error, .. } = current else {
panic!("late completion must fail");
};
assert_eq!(error.message, FINAL_TASK_ATTEMPT_DEADLINE_EXCEEDED);
} else {
assert!(matches!(current, FinalTask::Completed { .. }));
}
}
assert!(runtime.is_task_service_ready());
assert_final_task_payload_accounting(&store);
drop(running);
});
}
#[test]
fn task_02_deadline_inside_application_poll_rejects_late_completion() {
check_execution_completion_boundary(true, false);
}
#[test]
fn task_02_live_deadline_inside_application_poll_accepts_completion() {
check_execution_completion_boundary(false, false);
}
#[test]
fn task_02_committed_success_cleanup_stops_at_attempt_deadline() {
check_execution_completion_boundary(true, true);
}
#[test]
fn task_02_committed_success_cleanup_remains_live_before_deadline() {
check_execution_completion_boundary(false, true);
}
struct RetentionExpiryDropFlag(Arc<AtomicBool>);
impl Drop for RetentionExpiryDropFlag {
fn drop(&mut self) {
self.0.store(true, AtomicOrdering::SeqCst);
}
}
struct RetentionPollSupervisor {
polls: Arc<AtomicUsize>,
dropped: Arc<AtomicBool>,
}
impl ApplicationTaskSupervisor for RetentionPollSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
let _handoff = handoff;
let _lifetime = RetentionExpiryDropFlag(Arc::clone(&self.dropped));
std::future::poll_fn(|_| {
self.polls.fetch_add(1, AtomicOrdering::SeqCst);
std::task::Poll::Pending::<McpResult<()>>
})
.await
})
}
}
fn check_retention_before_supervisor_repoll(expired: bool) {
let (store, clock) = in_memory_store_with_test_clock(1);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_ttl(Some(1_000), None).expect("finite retention"),
Arc::new(|_| {}),
);
let polls = Arc::new(AtomicUsize::new(0));
let dropped = Arc::new(AtomicBool::new(false));
let mut runner = runtime
.install_task_service(
1,
Arc::new(RetentionPollSupervisor {
polls: Arc::clone(&polls),
dropped: Arc::clone(&dropped),
}),
)
.expect("install real pending application work");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("caller-owned runtime");
application_runtime.block_on(async {
let cx = Cx::current().expect("caller execution context");
let mut running = Box::pin(runner.run_service(&cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(running.as_mut().poll(&mut context).is_pending());
assert!(
runtime.is_task_service_ready(),
"service entered before task creation"
);
let created = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("create retained task");
let task_id = created.task.base().task_id.clone();
assert!(running.as_mut().poll(&mut context).is_pending());
assert_eq!(polls.load(AtomicOrdering::SeqCst), 1);
assert!(!dropped.load(AtomicOrdering::SeqCst));
let before = store.get_task(&task_id).expect("retained task read");
{
let mut now = clock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*now += StdDuration::from_millis(if expired { 1_001 } else { 999 });
}
assert!(running.as_mut().poll(&mut context).is_pending());
if expired {
assert_eq!(
polls.load(AtomicOrdering::SeqCst),
1,
"expired application work must be dropped before another poll"
);
assert!(dropped.load(AtomicOrdering::SeqCst));
assert!(
store
.get_task(&task_id)
.expect("expired task read")
.is_none()
);
assert_eq!(store.task_count(), 0);
} else {
assert_eq!(polls.load(AtomicOrdering::SeqCst), 2);
assert!(!dropped.load(AtomicOrdering::SeqCst));
assert_eq!(
serde_json::to_value(store.get_task(&task_id).expect("unexpired task read"))
.expect("serialize unexpired task"),
serde_json::to_value(before).expect("serialize original task")
);
assert_eq!(store.task_count(), 1);
}
drop(running);
assert!(dropped.load(AtomicOrdering::SeqCst));
});
}
#[test]
fn task_02_expired_handoff_is_not_repolled() {
check_retention_before_supervisor_repoll(true);
}
#[test]
fn task_02_unexpired_handoff_is_repolled() {
check_retention_before_supervisor_repoll(false);
}
fn check_dispatch_lease_before_supervisor_repoll(expired: bool, resumed: bool) {
let (store, clock) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let inputs: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}})).unwrap();
let task_id = if resumed {
create_accepted_final_input(&runtime, inputs.clone())
} else {
create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone()
};
let handoff = if resumed {
FinalTaskSupervisorHandoff::Resumed(
runtime.take_accepted_input(&task_id).unwrap().unwrap(),
)
} else {
FinalTaskSupervisorHandoff::Initial(runtime.recover_initial_work().unwrap().unwrap())
};
let polls = Arc::new(AtomicUsize::new(0));
let dropped = Arc::new(AtomicBool::new(false));
let runner = runtime
.install_task_service(
1,
Arc::new(RetentionPollSupervisor {
polls: Arc::clone(&polls),
dropped: Arc::clone(&dropped),
}),
)
.unwrap();
let cx = Cx::for_testing();
let mut running = runner.resume_handoff(&cx, handoff);
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(running.as_mut().poll(&mut context).is_pending());
assert_eq!(polls.load(AtomicOrdering::SeqCst), 1);
let before = store.get_task_snapshot(&task_id).unwrap().unwrap();
*clock.lock().unwrap() += if expired {
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE
} else {
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE - StdDuration::from_millis(1)
};
let outcome = running.as_mut().poll(&mut context);
if expired {
assert!(matches!(outcome, std::task::Poll::Ready(Err(_))));
assert_eq!(
polls.load(AtomicOrdering::SeqCst),
1,
"an expired dispatch owner cannot poll application work again"
);
assert!(dropped.load(AtomicOrdering::SeqCst));
let recovered = store.get_task_snapshot(&task_id).unwrap().unwrap();
assert!(recovered.generation() > before.generation());
assert_eq!(
serde_json::to_value(recovered.task()).unwrap(),
serde_json::to_value(before.task()).unwrap(),
"lease loss retains the working task and its retention contract"
);
if resumed {
let claim = store
.take_input_handoff_for_owner_if_current(&recovered, "successor")
.unwrap()
.unwrap();
assert_eq!(claim.input_responses, inputs);
assert_eq!(claim.work_descriptor, final_test_work_descriptor());
} else {
let claim = store
.take_initial_work_handoff_for_owner_if_current(&recovered, "successor")
.unwrap()
.unwrap();
assert_eq!(claim.work_descriptor, final_test_work_descriptor());
}
let successor_fence = store
.begin_handoff_dispatch_for_owner_if_current(
&task_id,
recovered.generation(),
"successor",
)
.unwrap()
.unwrap();
drop(running);
let state = store.state.lock().unwrap();
let lease = state.handoff_leases.get(&task_id).unwrap();
assert_eq!(lease.owner_id, "successor");
assert_eq!(lease.dispatch_fence, Some(successor_fence));
} else {
assert!(outcome.is_pending());
assert_eq!(polls.load(AtomicOrdering::SeqCst), 2);
assert!(!dropped.load(AtomicOrdering::SeqCst));
let current = store.get_task_snapshot(&task_id).unwrap().unwrap();
assert_eq!(current.generation(), before.generation());
assert_eq!(
serde_json::to_value(current.task()).unwrap(),
serde_json::to_value(before.task()).unwrap()
);
drop(running);
assert!(dropped.load(AtomicOrdering::SeqCst));
assert!(
!store
.state
.lock()
.unwrap()
.handoff_leases
.contains_key(&task_id)
);
}
assert_eq!(store.task_count(), 1);
}
#[test]
fn task_02_expired_initial_dispatch_lease_stops_before_application_repoll() {
check_dispatch_lease_before_supervisor_repoll(true, false);
}
#[test]
fn task_02_live_initial_dispatch_lease_allows_application_repoll() {
check_dispatch_lease_before_supervisor_repoll(false, false);
}
#[test]
fn task_02_expired_resumed_dispatch_lease_stops_before_application_repoll() {
check_dispatch_lease_before_supervisor_repoll(true, true);
}
#[test]
fn task_02_live_resumed_dispatch_lease_allows_application_repoll() {
check_dispatch_lease_before_supervisor_repoll(false, true);
}
struct DispatchTransitionCleanupSupervisor {
owns_transition: bool,
polls: Arc<AtomicUsize>,
dropped: Arc<AtomicBool>,
}
impl ApplicationTaskSupervisor for DispatchTransitionCleanupSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async move {
let _lifetime = RetentionExpiryDropFlag(Arc::clone(&self.dropped));
let FinalTaskSupervisorHandoff::Initial(initial) = handoff else {
panic!("the cleanup fixture requires an initial handoff");
};
if self.owns_transition {
initial.require_input(final_roots_request(), None)?;
}
std::future::poll_fn(|_| {
self.polls.fetch_add(1, AtomicOrdering::SeqCst);
std::task::Poll::Pending::<McpResult<()>>
})
.await
})
}
}
fn check_cleanup_after_dispatch_transition(owns_transition: bool) {
let (store, clock) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let handoff =
FinalTaskSupervisorHandoff::Initial(runtime.recover_initial_work().unwrap().unwrap());
let polls = Arc::new(AtomicUsize::new(0));
let dropped = Arc::new(AtomicBool::new(false));
let runner = runtime
.install_task_service(
1,
Arc::new(DispatchTransitionCleanupSupervisor {
owns_transition,
polls: Arc::clone(&polls),
dropped: Arc::clone(&dropped),
}),
)
.unwrap();
let cx = Cx::for_testing();
let mut running = runner.resume_handoff(&cx, handoff);
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(running.as_mut().poll(&mut context).is_pending());
assert_eq!(polls.load(AtomicOrdering::SeqCst), 1);
if !owns_transition {
runtime
.require_input(&task_id, final_roots_request(), None)
.unwrap();
}
let before = final_task_restoration_snapshot(&store, &task_id);
assert!(matches!(
store.get_task(&task_id).unwrap(),
Some(FinalTask::InputRequired { .. })
));
assert!(
!store
.state
.lock()
.unwrap()
.handoff_leases
.contains_key(&task_id)
);
*clock.lock().unwrap() += IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
let outcome = running.as_mut().poll(&mut context);
if owns_transition {
assert!(outcome.is_pending());
assert_eq!(polls.load(AtomicOrdering::SeqCst), 2);
assert!(!dropped.load(AtomicOrdering::SeqCst));
} else {
assert!(matches!(outcome, std::task::Poll::Ready(Err(_))));
assert_eq!(polls.load(AtomicOrdering::SeqCst), 1);
assert!(dropped.load(AtomicOrdering::SeqCst));
}
drop(running);
assert!(dropped.load(AtomicOrdering::SeqCst));
assert_eq!(final_task_restoration_snapshot(&store, &task_id), before);
}
#[test]
fn task_02_committed_handoff_transition_keeps_cleanup_pollable() {
check_cleanup_after_dispatch_transition(true);
}
#[test]
fn task_02_foreign_transition_does_not_authorize_stale_cleanup() {
check_cleanup_after_dispatch_transition(false);
}
struct ExpiryThenCompletingSupervisor {
runtime: FinalTaskRuntime,
clock: Arc<Mutex<Instant>>,
task1_id: FinalTaskId,
task1_dropped: Arc<AtomicBool>,
task2_id_slot: Arc<Mutex<Option<FinalTaskId>>>,
task2_completed: Arc<AtomicBool>,
}
impl ApplicationTaskSupervisor for ExpiryThenCompletingSupervisor {
fn resume<'a>(
&'a self,
cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let runtime = self.runtime.clone();
let clock = Arc::clone(&self.clock);
let task1_id = self.task1_id.clone();
let task1_dropped = Arc::clone(&self.task1_dropped);
let task2_id_slot = Arc::clone(&self.task2_id_slot);
let task2_completed = Arc::clone(&self.task2_completed);
let current_id = final_task_handoff_task_id(&handoff).clone();
Box::pin(async move {
if current_id == task1_id {
let _drop_guard = RetentionExpiryDropFlag(task1_dropped);
{
let mut clk = clock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clk = clk
.checked_add(StdDuration::from_secs(30))
.expect("advance test clock partway");
}
let created2 = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("finite-TTL task 2 is created while service is ready");
let task2_id = created2.task.base().task_id.clone();
*task2_id_slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(task2_id);
{
let mut clk = clock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clk = clk
.checked_add(StdDuration::from_secs(35))
.expect("advance test clock past task 1 expiry");
}
std::future::pending::<McpResult<()>>().await
} else {
assert!(
task1_dropped.load(AtomicOrdering::SeqCst),
"supervisor future for task 1 must be dropped before task 2 invocation"
);
let expected_task2_id = task2_id_slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
.expect("task 2 id must have been recorded during task 1 execution");
assert_eq!(
current_id, expected_task2_id,
"second invocation must execute task 2"
);
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
match handoff {
FinalTaskSupervisorHandoff::Initial(initial) => {
initial
.complete_task(result, None)
.expect("complete task 2");
}
FinalTaskSupervisorHandoff::Resumed(accepted) => {
accepted
.complete_task(result, None)
.expect("complete task 2");
}
}
task2_completed.store(true, AtomicOrdering::SeqCst);
cx.cancel_with(CancelKind::User, None);
Ok(())
}
})
}
}
struct UnexpiredLostLeaseSupervisor {
store: Arc<InMemoryFinalTaskStore>,
task_id: FinalTaskId,
}
impl ApplicationTaskSupervisor for UnexpiredLostLeaseSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let store = Arc::clone(&self.store);
let task_id = self.task_id.clone();
assert_eq!(final_task_handoff_task_id(&handoff), &task_id);
Box::pin(async move {
store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.handoff_leases
.remove(&task_id);
std::future::pending::<McpResult<()>>().await
})
}
}
struct FastHeartbeatProbeStore {
inner: Arc<InMemoryFinalTaskStore>,
heartbeat: StdDuration,
reject_cancelled_renewal: bool,
}
impl FinalTaskStore for FastHeartbeatProbeStore {
delegate_final_execution_policy!();
fn next_accepted_input_snapshot_after(
&self,
after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_accepted_input_snapshot_after(after_task_id)
}
fn create_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.create_task(task, notification)
}
fn create_task_with_work(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<FinalTaskSnapshot> {
self.inner
.create_task_with_work(task, notification, work_descriptor)
}
fn get_task(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTask>> {
self.inner.get_task(task_id)
}
fn get_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.get_task_snapshot(task_id)
}
fn replace_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.replace_task(task, notification)
}
fn replace_task_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<bool> {
self.inner
.replace_task_if_current(expected, task, notification)
}
fn request_cancellation(&self, task_id: &FinalTaskId) -> McpResult<()> {
self.inner.request_cancellation(task_id)
}
fn request_cancellation_if_current(&self, expected: &FinalTaskSnapshot) -> McpResult<bool> {
self.inner.request_cancellation_if_current(expected)
}
fn is_cancellation_requested(&self, task_id: &FinalTaskId) -> McpResult<bool> {
self.inner.is_cancellation_requested(task_id)
}
fn next_initial_work_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_initial_work_snapshot()
}
fn next_initial_work_snapshot_after(
&self,
after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_initial_work_snapshot_after(after_task_id)
}
fn take_initial_work_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
self.inner.take_initial_work_if_current(expected)
}
fn take_initial_work_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
self.inner
.take_initial_work_for_owner_if_current(expected, owner_id)
}
fn take_initial_work_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskInitialWorkClaim>> {
self.inner
.take_initial_work_handoff_for_owner_if_current(expected, owner_id)
}
fn restore_initial_work_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
self.inner
.restore_initial_work_if_current(task_id, generation, work_descriptor)
}
fn restore_initial_work_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: Option<u64>,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
self.inner.restore_initial_work_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
work_descriptor,
)
}
fn replace_task_and_append_input_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
input_responses: FinalTaskInputResponses,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.replace_task_and_append_input_if_current(
expected,
task,
notification,
input_responses,
)
}
fn replace_task_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner
.replace_task_and_clear_input_if_current(expected, task, notification)
}
fn replace_task_and_clear_input_for_handoff_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
dispatch_fence: u64,
cancellation_required: bool,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner
.replace_task_and_clear_input_for_handoff_if_current(
expected,
owner_id,
dispatch_fence,
cancellation_required,
task,
notification,
)
}
fn begin_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<bool> {
self.inner
.begin_handoff_dispatch_if_current(task_id, generation)
}
fn begin_handoff_dispatch_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
) -> McpResult<Option<u64>> {
self.inner
.begin_handoff_dispatch_for_owner_if_current(task_id, generation, owner_id)
}
fn renew_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
if self.reject_cancelled_renewal && self.inner.is_cancellation_requested(task_id)? {
return Ok(false);
}
self.inner.renew_handoff_dispatch_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
)
}
fn handoff_dispatch_lease_heartbeat_interval(&self) -> McpResult<StdDuration> {
Ok(self.heartbeat)
}
fn finish_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<bool> {
self.inner
.finish_handoff_dispatch_if_current(task_id, generation)
}
fn finish_handoff_dispatch_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
self.inner.finish_handoff_dispatch_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
)
}
fn request_cancellation_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
cancelled_task: FinalTask,
cancelled_notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.request_cancellation_and_clear_input_if_current(
expected,
cancelled_task,
cancelled_notification,
)
}
fn retention_clock_now(&self) -> Instant {
self.inner.retention_clock_now()
}
fn task_retention_deadline_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>> {
self.inner
.task_retention_deadline_if_current(task_id, generation)
}
}
fn check_cancellation_after_dispatch_renewal(reject_cancelled_renewal: bool) {
let (inner, _clock) = in_memory_store_with_test_clock(1);
let store = Arc::new(FastHeartbeatProbeStore {
inner: Arc::clone(&inner),
heartbeat: IN_MEMORY_FINAL_TASK_HANDOFF_HEARTBEAT,
reject_cancelled_renewal,
});
let runtime = FinalTaskRuntime::new(
store,
FinalTaskRuntimeConfig::new(60_000, None).unwrap(),
Arc::new(|_| {}),
);
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let handoff =
FinalTaskSupervisorHandoff::Initial(runtime.recover_initial_work().unwrap().unwrap());
let polls = Arc::new(AtomicUsize::new(0));
let dropped = Arc::new(AtomicBool::new(false));
let runner = runtime
.install_task_service(
1,
Arc::new(RetentionPollSupervisor {
polls: Arc::clone(&polls),
dropped: Arc::clone(&dropped),
}),
)
.unwrap();
let cx = Cx::for_testing();
let mut running = runner.resume_handoff(&cx, handoff);
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(running.as_mut().poll(&mut context).is_pending());
assert_eq!(polls.load(AtomicOrdering::SeqCst), 1);
runtime.cancel_task(&task_id).unwrap();
assert!(inner.is_cancellation_requested(&task_id).unwrap());
let outcome = running.as_mut().poll(&mut context);
if reject_cancelled_renewal {
assert!(matches!(outcome, std::task::Poll::Ready(Ok(()))));
assert_eq!(
polls.load(AtomicOrdering::SeqCst),
1,
"a refused renewal cannot grant the final cancellation poll"
);
} else {
assert!(outcome.is_pending());
assert!(polls.load(AtomicOrdering::SeqCst) > 1);
assert!(matches!(
running.as_mut().poll(&mut context),
std::task::Poll::Ready(Ok(()))
));
}
assert!(dropped.load(AtomicOrdering::SeqCst));
drop(running);
assert!(matches!(
inner.get_task(&task_id).unwrap(),
Some(FinalTask::Cancelled(_))
));
assert!(!inner.is_cancellation_requested(&task_id).unwrap());
let state = inner.state.lock().unwrap();
assert!(!state.handoff_leases.contains_key(&task_id));
assert!(!state.initial_work.contains_key(&task_id));
assert!(!state.accepted_inputs.contains_key(&task_id));
}
#[test]
fn task_02_refused_cancelled_renewal_retires_without_application_repoll() {
check_cancellation_after_dispatch_renewal(true);
}
#[test]
fn task_02_live_cancelled_lease_preserves_application_cancellation_window() {
check_cancellation_after_dispatch_renewal(false);
}
fn assert_queued_wakeup_progress(elapsed_ms: u64, first_should_execute: bool) {
let (store, now) = in_memory_store_with_test_clock(2);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(1_000, None).expect("finite retention policy is valid"),
Arc::new(|_| {}),
);
let started = Arc::new(Mutex::new(Vec::new()));
let mut runner = runtime
.install_task_service(
2,
Arc::new(RecordingQueuedWakeupSupervisor {
started: Arc::clone(&started),
}),
)
.expect("install caller-owned queued wakeup service");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("caller-owned runtime builds");
application_runtime.block_on(async {
let cx = Cx::current().expect("caller runtime supplies service context");
let mut service = Box::pin(runner.run_service(&cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
Future::poll(service.as_mut(), &mut context),
std::task::Poll::Pending
));
let first = runtime
.create_task_with_work(
FinalTaskWorkDescriptor::new(serde_json::json!({"label": "first"}))
.expect("first work descriptor is valid"),
None,
)
.expect("ready service accepts first task")
.task
.base()
.task_id
.clone();
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
StdDuration::from_millis(elapsed_ms);
let second = runtime
.create_task_with_work(
FinalTaskWorkDescriptor::new(serde_json::json!({"label": "second"}))
.expect("second work descriptor is valid"),
None,
)
.expect("ready service accepts second task before dequeuing the first wakeup")
.task
.base()
.task_id
.clone();
assert!(matches!(
Future::poll(service.as_mut(), &mut context),
std::task::Poll::Pending
));
assert!(
runtime.is_task_service_ready(),
"processing advisory wakeups preserves the live service generation"
);
let expected = if first_should_execute {
vec![first.clone(), second.clone()]
} else {
vec![second.clone()]
};
assert_eq!(
*started
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
expected,
"every retained task executes once; expired work never reaches the supervisor"
);
for (task_id, label) in [(&first, "first"), (&second, "second")] {
if task_id == &first && !first_should_execute {
assert!(
store
.get_task(task_id)
.expect("expired task lookup succeeds")
.is_none()
);
continue;
}
let task = runtime
.get_task(task_id)
.expect("completed task is readable")
.task;
let FinalTask::Completed { result, .. } = task else {
panic!("retained queued task did not commit its fenced completion");
};
assert_eq!(
serde_json::to_value(result).expect("completed result serializes")["content"],
serde_json::json!([{"type": "text", "text": label}])
);
}
cx.cancel_with(CancelKind::User, None);
assert!(matches!(
Future::poll(service.as_mut(), &mut context),
std::task::Poll::Ready(Ok(()))
));
assert!(!runtime.is_task_service_ready());
});
}
#[test]
fn task_03_final_expired_queued_wakeup_preserves_service_progress() {
assert_queued_wakeup_progress(1_000, false);
}
#[test]
fn task_03_final_unexpired_queued_wakeup_preserves_service_progress() {
assert_queued_wakeup_progress(999, true);
}
#[test]
fn task_03_final_service_runner_recovers_after_dispatch_lease_expiry_without_event() {
let (inner_store, now) = in_memory_store_with_test_clock(1);
let runtime = FinalTaskRuntime::new(
inner_store.clone(),
FinalTaskRuntimeConfig::new(60_000, None).expect("valid final task policy"),
Arc::new(|_| {}),
);
let task = final_working_task_without_ttl("task-service-expired-lease-wakeup");
let task_id = task.base().task_id.clone();
let work_descriptor = final_test_work_descriptor();
inner_store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("predecessor task work is durably retained");
let snapshot = inner_store
.get_task_snapshot(&task_id)
.expect("predecessor task snapshot is readable")
.expect("predecessor task remains retained");
let claim = inner_store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "predecessor")
.expect("predecessor claim is readable")
.expect("predecessor claims the initial work");
inner_store
.begin_handoff_dispatch_for_owner_if_current(&task_id, claim.generation, "predecessor")
.expect("predecessor dispatch election is readable")
.expect("predecessor wins dispatch election");
let started = Arc::new(Mutex::new(Vec::new()));
let mut runner = runtime
.install_task_service(
1,
Arc::new(RecordingInitialFinalTaskSupervisor {
started: Arc::clone(&started),
}),
)
.expect("successor task service installs");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("caller-owned runtime builds");
let result = application_runtime
.block_on(async {
let cx = Cx::current().expect("caller runtime supplies service context");
let mut service = Box::pin(runner.run_service(&cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
Future::poll(service.as_mut(), &mut context),
std::task::Poll::Pending
));
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE;
asupersync::time::timeout(cx.now(), StdDuration::from_secs(5), service).await
})
.expect("periodic recovery wake completes within the bounded timeout");
result.expect("successor service recovers and completes the retained task");
assert_eq!(
started
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_slice(),
&[(task_id.clone(), work_descriptor)],
"the timer wake delivers the exact retained work without a task event"
);
let committed = inner_store
.get_task(&task_id)
.expect("completed task remains readable")
.expect("completed task remains retained");
assert!(
matches!(committed, FinalTask::Completed { .. }),
"successor recovery commits the supervisor result"
);
let state = inner_store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(!state.handoff_leases.contains_key(&task_id));
assert!(!state.initial_work.contains_key(&task_id));
}
#[test]
fn task_03_final_service_runner_does_not_recover_live_dispatch_lease_before_expiry() {
let (inner_store, now) = in_memory_store_with_test_clock(1);
let runtime = FinalTaskRuntime::new(
inner_store.clone(),
FinalTaskRuntimeConfig::new(60_000, None).expect("valid final task policy"),
Arc::new(|_| {}),
);
let task = final_working_task_without_ttl("task-service-live-lease-wakeup");
let task_id = task.base().task_id.clone();
let work_descriptor = final_test_work_descriptor();
inner_store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("predecessor task work is durably retained");
let snapshot = inner_store
.get_task_snapshot(&task_id)
.expect("predecessor task snapshot is readable")
.expect("predecessor task remains retained");
let claim = inner_store
.take_initial_work_handoff_for_owner_if_current(&snapshot, "predecessor")
.expect("predecessor claim is readable")
.expect("predecessor claims the initial work");
let dispatch_fence = inner_store
.begin_handoff_dispatch_for_owner_if_current(&task_id, claim.generation, "predecessor")
.expect("predecessor dispatch election is readable")
.expect("predecessor wins dispatch election");
let started = Arc::new(Mutex::new(Vec::new()));
let mut runner = runtime
.install_task_service(
1,
Arc::new(RecordingInitialFinalTaskSupervisor {
started: Arc::clone(&started),
}),
)
.expect("successor task service installs");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("caller-owned runtime builds");
let timed = application_runtime.block_on(async {
let cx = Cx::current().expect("caller runtime supplies service context");
let mut service = Box::pin(runner.run_service(&cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
Future::poll(service.as_mut(), &mut context),
std::task::Poll::Pending
));
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE
.checked_sub(StdDuration::from_millis(1))
.expect("handoff lease exceeds one millisecond");
asupersync::time::timeout(cx.now(), StdDuration::from_secs(2), service).await
});
assert!(
timed.is_err(),
"a periodic scan must not recover a dispatch lease before expiry"
);
assert!(
started
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty(),
"the live predecessor lease prevents successor supervisor invocation"
);
let retained = inner_store
.get_task(&task_id)
.expect("live task remains readable")
.expect("live task remains retained");
assert_eq!(
serde_json::to_value(retained).expect("serialize retained live task"),
serde_json::to_value(task).expect("serialize original live task")
);
let state = inner_store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
state.generations.get(&task_id),
Some(&snapshot.generation())
);
assert_eq!(state.initial_work.get(&task_id), Some(&work_descriptor));
let lease = state
.handoff_leases
.get(&task_id)
.expect("live predecessor lease remains retained");
assert_eq!(lease.owner_id, "predecessor");
assert_eq!(lease.dispatch_fence, Some(dispatch_fence));
assert!(lease.recovery_expires_at.is_some_and(|expires_at| {
expires_at
> *now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}));
}
#[test]
fn task_03_final_runtime_worker_quiesces_on_retention_expiry_and_proceeds_to_next_task() {
const TTL_MS: u64 = 60_000;
let (store, now) = in_memory_store_with_test_clock(2);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_ttl(Some(TTL_MS), None)
.expect("positive TTL is a valid Task retention value"),
Arc::new(|_| {}),
);
let dummy_runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install initial dummy runner to create tasks");
let dummy_cx = Cx::for_testing();
let running_service = enter_task_service_runner(dummy_runner, &dummy_cx);
let created1 = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("finite-TTL task 1 is durably created");
let task_id1 = created1.task.base().task_id.clone();
drop(running_service);
let task1_dropped = Arc::new(AtomicBool::new(false));
let task2_id_slot = Arc::new(Mutex::new(None));
let task2_completed = Arc::new(AtomicBool::new(false));
let supervisor = Arc::new(ExpiryThenCompletingSupervisor {
runtime: runtime.clone(),
clock: Arc::clone(&now),
task1_id: task_id1.clone(),
task1_dropped: Arc::clone(&task1_dropped),
task2_id_slot: Arc::clone(&task2_id_slot),
task2_completed: Arc::clone(&task2_completed),
});
let mut runner = runtime
.install_task_service(4, supervisor)
.expect("install test supervisor runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
let timeout_duration = StdDuration::from_secs(5);
let run_outcome = application_runtime
.block_on(async {
let service_cx =
Cx::current().expect("the caller runtime supplies the service context");
asupersync::time::timeout(
service_cx.now(),
timeout_duration,
runner.run_service(&service_cx),
)
.await
})
.expect("service runner must finish within finite timeout");
run_outcome.expect("service runner completes without aborting on task 1 expiry");
assert!(
task1_dropped.load(AtomicOrdering::SeqCst),
"supervisor future for task 1 must be dropped and quiesced upon retention expiry"
);
assert!(
task2_completed.load(AtomicOrdering::SeqCst),
"the runner must proceed to execute and complete the second task in the backlog"
);
let task_id2 = task2_id_slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
.expect("task 2 id was captured during execution");
assert!(
runtime.get_task(&task_id1).is_err(),
"task 1 was purged upon retention expiry without aborting the runner"
);
assert!(
store.get_task(&task_id1).expect("store read").is_none(),
"task 1 is deleted from the durable store"
);
assert!(
matches!(
store.get_task(&task_id2).expect("store read"),
Some(FinalTask::Completed { .. })
),
"task 2 remains committed as Completed in the store"
);
assert_eq!(
store.task_count(),
1,
"task 1 retention expiry freed its store quota while task 2 remains retained"
);
}
#[test]
fn task_03_final_runtime_worker_unexpired_lost_lease_fails() {
const TTL_MS: u64 = 60_000;
let (inner_store, _now) = in_memory_store_with_test_clock(1);
let store = Arc::new(FastHeartbeatProbeStore {
inner: Arc::clone(&inner_store),
heartbeat: StdDuration::from_millis(1),
reject_cancelled_renewal: false,
});
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_ttl(Some(TTL_MS), None)
.expect("positive TTL is a valid Task retention value"),
Arc::new(|_| {}),
);
let dummy_runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install initial dummy runner to create tasks");
let dummy_cx = Cx::for_testing();
let running_service = enter_task_service_runner(dummy_runner, &dummy_cx);
let created = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("finite-TTL task is durably created");
let task_id = created.task.base().task_id.clone();
drop(running_service);
let supervisor = Arc::new(UnexpiredLostLeaseSupervisor {
store: Arc::clone(&inner_store),
task_id: task_id.clone(),
});
let runner = runtime
.install_task_service(2, supervisor)
.expect("install test supervisor runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
let timeout_duration = StdDuration::from_secs(5);
let result = application_runtime
.block_on(async {
let test_cx =
Cx::current().expect("the caller runtime supplies the execution context");
let initial = runtime
.recover_initial_work_with_checkpoints(&test_cx, &runner.dispatch_owner, None)
.expect("recover initial handoff")
.expect("initial handoff is present");
asupersync::time::timeout(
test_cx.now(),
timeout_duration,
runner.resume_handoff(&test_cx, FinalTaskSupervisorHandoff::Initial(initial)),
)
.await
})
.expect("resume_handoff must complete within finite timeout");
let error = result.expect_err("an unexpired lost lease must fail strictly");
assert!(
error
.to_string()
.contains("Final task dispatch lease was lost while application work was running"),
"unexpected error message: {error}"
);
assert!(
runtime.get_task(&task_id).is_ok(),
"unexpired task with lost lease must remain in the store"
);
assert!(
matches!(
inner_store.get_task(&task_id).expect("store read"),
Some(FinalTask::Working(_))
),
"unexpired task remains in Working state"
);
assert_eq!(
inner_store.task_count(),
1,
"unexpired task retains its store quota"
);
}
struct FailingSnapshotProbeStore {
inner: Arc<InMemoryFinalTaskStore>,
fail_snapshots: AtomicBool,
advance_clock_on_failure: Option<(Arc<Mutex<Instant>>, u64)>,
}
impl FinalTaskStore for FailingSnapshotProbeStore {
delegate_final_execution_policy!();
fn next_accepted_input_snapshot_after(
&self,
after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_accepted_input_snapshot_after(after_task_id)
}
fn create_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.create_task(task, notification)
}
fn create_task_with_work(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<FinalTaskSnapshot> {
self.inner
.create_task_with_work(task, notification, work_descriptor)
}
fn get_task(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTask>> {
self.inner.get_task(task_id)
}
fn get_task_snapshot(&self, task_id: &FinalTaskId) -> McpResult<Option<FinalTaskSnapshot>> {
if self.fail_snapshots.load(AtomicOrdering::SeqCst) {
if let Some((clock, advance_ms)) = &self.advance_clock_on_failure {
let mut clk = clock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clk = clk
.checked_add(StdDuration::from_millis(*advance_ms))
.expect("advance test clock");
}
return Err(McpError::internal_error(
"simulated durable backend failure",
));
}
self.inner.get_task_snapshot(task_id)
}
fn replace_task(
&self,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<()> {
self.inner.replace_task(task, notification)
}
fn replace_task_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<bool> {
self.inner
.replace_task_if_current(expected, task, notification)
}
fn request_cancellation(&self, task_id: &FinalTaskId) -> McpResult<()> {
self.inner.request_cancellation(task_id)
}
fn request_cancellation_if_current(&self, expected: &FinalTaskSnapshot) -> McpResult<bool> {
self.inner.request_cancellation_if_current(expected)
}
fn is_cancellation_requested(&self, task_id: &FinalTaskId) -> McpResult<bool> {
self.inner.is_cancellation_requested(task_id)
}
fn next_initial_work_snapshot(&self) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_initial_work_snapshot()
}
fn next_initial_work_snapshot_after(
&self,
after_task_id: Option<&FinalTaskId>,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.next_initial_work_snapshot_after(after_task_id)
}
fn take_initial_work_if_current(
&self,
expected: &FinalTaskSnapshot,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
self.inner.take_initial_work_if_current(expected)
}
fn take_initial_work_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskWorkDescriptor>> {
self.inner
.take_initial_work_for_owner_if_current(expected, owner_id)
}
fn take_initial_work_handoff_for_owner_if_current(
&self,
expected: &FinalTaskSnapshot,
owner_id: &str,
) -> McpResult<Option<FinalTaskInitialWorkClaim>> {
self.inner
.take_initial_work_handoff_for_owner_if_current(expected, owner_id)
}
fn restore_initial_work_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
self.inner
.restore_initial_work_if_current(task_id, generation, work_descriptor)
}
fn restore_initial_work_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: Option<u64>,
work_descriptor: FinalTaskWorkDescriptor,
) -> McpResult<bool> {
self.inner.restore_initial_work_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
work_descriptor,
)
}
fn replace_task_and_append_input_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
input_responses: FinalTaskInputResponses,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.replace_task_and_append_input_if_current(
expected,
task,
notification,
input_responses,
)
}
fn replace_task_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
task: FinalTask,
notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner
.replace_task_and_clear_input_if_current(expected, task, notification)
}
fn begin_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<bool> {
self.inner
.begin_handoff_dispatch_if_current(task_id, generation)
}
fn begin_handoff_dispatch_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
) -> McpResult<Option<u64>> {
self.inner
.begin_handoff_dispatch_for_owner_if_current(task_id, generation, owner_id)
}
fn renew_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
self.inner.renew_handoff_dispatch_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
)
}
fn handoff_dispatch_lease_heartbeat_interval(&self) -> McpResult<StdDuration> {
self.inner.handoff_dispatch_lease_heartbeat_interval()
}
fn finish_handoff_dispatch_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<bool> {
self.inner
.finish_handoff_dispatch_if_current(task_id, generation)
}
fn finish_handoff_dispatch_for_owner_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
owner_id: &str,
dispatch_fence: u64,
) -> McpResult<bool> {
self.inner.finish_handoff_dispatch_for_owner_if_current(
task_id,
generation,
owner_id,
dispatch_fence,
)
}
fn request_cancellation_and_clear_input_if_current(
&self,
expected: &FinalTaskSnapshot,
cancelled_task: FinalTask,
cancelled_notification: FinalTaskStatusNotification,
) -> McpResult<Option<FinalTaskSnapshot>> {
self.inner.request_cancellation_and_clear_input_if_current(
expected,
cancelled_task,
cancelled_notification,
)
}
fn retention_clock_now(&self) -> Instant {
self.inner.retention_clock_now()
}
fn task_retention_deadline_if_current(
&self,
task_id: &FinalTaskId,
generation: u64,
) -> McpResult<Option<FinalTaskRetentionDeadline>> {
self.inner
.task_retention_deadline_if_current(task_id, generation)
}
}
#[test]
fn task_03_final_runtime_backend_snapshot_error_propagates_without_laundering() {
const TTL_MS: u64 = 60_000;
let (inner, _now) = in_memory_store_with_test_clock(1);
let store = Arc::new(FailingSnapshotProbeStore {
inner,
fail_snapshots: AtomicBool::new(false),
advance_clock_on_failure: None,
});
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_ttl(Some(TTL_MS), None)
.expect("positive TTL is a valid Task retention value"),
Arc::new(|_| {}),
);
let dummy_runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install initial dummy runner to create tasks");
let dummy_cx = Cx::for_testing();
let running_service = enter_task_service_runner(dummy_runner, &dummy_cx);
runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("create task");
drop(running_service);
let initial = runtime
.recover_initial_work_with_checkpoints(&Cx::for_testing(), "test-owner", None)
.expect("recover initial")
.expect("initial present");
let handoff = FinalTaskSupervisorHandoff::Initial(initial);
let mut guard = FinalTaskExecutionGuard::new(&runtime, "test-owner", &handoff);
assert!(guard.elect().expect("election succeeds"));
store.fail_snapshots.store(true, AtomicOrdering::SeqCst);
let cancellation_err = guard
.is_cancellation_requested()
.expect_err("backend error must propagate from is_cancellation_requested");
assert!(
cancellation_err
.to_string()
.contains("simulated durable backend failure"),
"backend error must not be laundered: {cancellation_err}"
);
let recoverable_err = guard
.is_recoverable_without_transition()
.expect_err("backend error must propagate from is_recoverable_without_transition");
assert!(
recoverable_err
.to_string()
.contains("simulated durable backend failure"),
"backend error must not be laundered: {recoverable_err}"
);
}
struct PendingSupervisorArmingBackendFailure {
store: Arc<FailingSnapshotProbeStore>,
}
impl ApplicationTaskSupervisor for PendingSupervisorArmingBackendFailure {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
_handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let store = Arc::clone(&self.store);
Box::pin(async move {
store.fail_snapshots.store(true, AtomicOrdering::SeqCst);
std::future::pending::<McpResult<()>>().await
})
}
}
#[test]
fn task_03_final_runtime_backend_snapshot_error_racing_expiry_propagates_through_runner() {
const TTL_MS: u64 = 60_000;
let (inner, now) = in_memory_store_with_test_clock(1);
let store = Arc::new(FailingSnapshotProbeStore {
inner: Arc::clone(&inner),
fail_snapshots: AtomicBool::new(false),
advance_clock_on_failure: Some((Arc::clone(&now), TTL_MS + 1000)),
});
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_ttl(Some(TTL_MS), None)
.expect("positive TTL is a valid Task retention value"),
Arc::new(|_| {}),
);
let dummy_runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install initial dummy runner to create tasks");
let dummy_cx = Cx::for_testing();
let running_service = enter_task_service_runner(dummy_runner, &dummy_cx);
let created = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("create task");
let task_id = created.task.base().task_id.clone();
drop(running_service);
let supervisor = Arc::new(PendingSupervisorArmingBackendFailure {
store: Arc::clone(&store),
});
let runner = runtime
.install_task_service(2, supervisor)
.expect("install test supervisor runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
let timeout_duration = StdDuration::from_secs(5);
let result = application_runtime
.block_on(async {
let test_cx =
Cx::current().expect("the caller runtime supplies the execution context");
let initial = runtime
.recover_initial_work_with_checkpoints(&test_cx, &runner.dispatch_owner, None)
.expect("recover initial")
.expect("initial present");
asupersync::time::timeout(
test_cx.now(),
timeout_duration,
runner.resume_handoff(&test_cx, FinalTaskSupervisorHandoff::Initial(initial)),
)
.await
})
.expect("resume_handoff must complete within finite timeout");
let error = result.expect_err(
"backend snapshot error racing expiry must propagate through runner without laundering into success",
);
assert!(
error
.to_string()
.contains("simulated durable backend failure"),
"backend error must be preserved: {error}"
);
assert!(
runtime.get_task(&task_id).is_err(),
"expired task must not be restored into working state after failure"
);
assert!(
inner.get_task(&task_id).expect("store read").is_none(),
"expired task was purged from the underlying store without revival"
);
}
#[test]
fn task_03_final_guard_elect_refuses_missing_or_stale_retention_evidence() {
const TTL_MS: u64 = 60_000;
let (store, _now) = in_memory_store_with_test_clock(1);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_ttl(Some(TTL_MS), None)
.expect("positive TTL is a valid Task retention value"),
Arc::new(|_| {}),
);
let dummy_runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install initial dummy runner to create tasks");
let dummy_cx = Cx::for_testing();
let running_service = enter_task_service_runner(dummy_runner, &dummy_cx);
let created = runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("create task");
let task_id = created.task.base().task_id.clone();
drop(running_service);
let retained = store
.get_task_snapshot(&task_id)
.expect("task snapshot is readable")
.expect("task is retained");
let stale_generation = retained.generation() + 99;
let mut stale_guard = FinalTaskExecutionGuard {
runtime: runtime.clone(),
task_id: task_id.clone(),
generation: stale_generation,
owner_id: "test-owner".to_owned(),
dispatch_fence: None,
transition_committed: Arc::new(AtomicBool::new(false)),
retention_deadline: None,
execution_budget: None,
restoration: None,
};
assert!(
!stale_guard.elect().expect("elect check succeeds"),
"elect must refuse stale generation without setting retention_deadline"
);
assert!(
stale_guard.retention_deadline.is_none(),
"retention deadline must remain None on refused election"
);
assert!(
stale_guard.dispatch_fence.is_none(),
"dispatch fence must remain None on refused election"
);
let missing_id = FinalTaskId::parse("missing-retention-evidence")
.expect("bounded missing task identifier");
assert!(store.get_task(&missing_id).expect("store read").is_none());
let mut missing_guard = FinalTaskExecutionGuard {
runtime: runtime.clone(),
task_id: missing_id.clone(),
generation: retained.generation(),
owner_id: "test-owner".to_owned(),
dispatch_fence: None,
transition_committed: Arc::new(AtomicBool::new(false)),
retention_deadline: None,
execution_budget: None,
restoration: None,
};
assert!(
!missing_guard
.elect()
.expect("missing task election is readable")
);
assert!(missing_guard.retention_deadline.is_none());
assert!(missing_guard.dispatch_fence.is_none());
assert!(store.get_task(&missing_id).expect("store read").is_none());
let after = store
.get_task_snapshot(&task_id)
.expect("retained task remains readable")
.expect("unrelated task remains present");
assert_eq!(after.generation(), retained.generation());
assert_eq!(
serde_json::to_value(after.task()).expect("retained task serializes"),
serde_json::to_value(retained.task()).expect("original task serializes"),
"refused elections cannot change the retained task"
);
assert_eq!(store.task_count(), 1);
}
fn final_input_task_with_keys(working: &FinalTask, keys: &[String]) -> FinalTask {
let mut base = working.base().clone();
base.status = FinalTaskStatus::InputRequired;
let roots = final_roots_request().into_values().next().unwrap();
FinalTask::InputRequired {
base,
input_requests: keys
.iter()
.map(|key| (key.clone(), roots.clone()))
.collect(),
}
}
fn input_key_store_snapshot(
store: &InMemoryFinalTaskStore,
task_id: &FinalTaskId,
) -> serde_json::Value {
let state = store.state.lock().unwrap();
serde_json::json!({
"task": state.tasks.get(task_id),
"generation": state.generations.get(task_id),
"next_generation": state.next_generation,
"notification": state.latest_notifications.get(task_id),
"accepted_inputs": state.accepted_inputs.get(task_id),
"history": state.input_key_history.get(task_id)
.map(|history| (&history.keys, history.key_bytes)),
"lease": state.handoff_leases.get(task_id).map(|lease| (
lease.generation, &lease.owner_id, lease.dispatch_elected, lease.dispatch_fence,
)),
"next_dispatch_fence": state.next_dispatch_fence,
"cancelled": state.cancellation_requests.contains(task_id),
"principal": state.authenticated_principals.get(task_id)
.map(|principal| principal.as_bytes()),
"task_count": state.tasks.len(),
})
}
fn assert_input_key_history_limit(count_limit: bool, over_limit: bool) {
let store = InMemoryFinalTaskStore::default();
let working = final_working_task_without_ttl("task-key-history-bound");
let task_id = working.base().task_id.clone();
store
.create_task(working.clone(), final_task_notification(&working))
.unwrap();
let earlier_keys: Vec<String> = if count_limit {
(0..MAX_IN_MEMORY_FINAL_TASK_INPUT_KEYS - 1)
.map(|index| format!("earlier-{index}"))
.collect()
} else {
vec!["a".repeat(MAX_IN_MEMORY_FINAL_TASK_INPUT_KEY_BYTES - 1)]
};
for keys in earlier_keys.chunks(fastmcp_protocol::MAX_TASK_INPUT_MAP_ENTRIES) {
let input = final_input_task_with_keys(&working, keys);
store
.replace_task(input.clone(), final_task_notification(&input))
.unwrap();
store
.replace_task(working.clone(), final_task_notification(&working))
.unwrap();
}
let before = input_key_store_snapshot(&store, &task_id);
let current = store.get_task_snapshot(&task_id).unwrap().unwrap();
let keys = if over_limit {
vec!["b".to_owned(), "c".to_owned()]
} else {
vec!["b".to_owned()]
};
let proposed = final_input_task_with_keys(&working, &keys);
let result = store.replace_task_if_current(
¤t,
proposed.clone(),
final_task_notification(&proposed),
);
if over_limit {
let error = result.expect_err("one extra key must exceed the selected lifetime bound");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
assert_eq!(
error.message,
if count_limit {
"Task input request key history count limit exceeded"
} else {
"Task input request key history byte limit exceeded"
}
);
assert_eq!(input_key_store_snapshot(&store, &task_id), before);
let admitted = final_input_task_with_keys(&working, &["b".to_owned()]);
assert!(
store
.replace_task_if_current(
¤t,
admitted.clone(),
final_task_notification(&admitted),
)
.unwrap()
);
} else {
assert!(result.expect("the exact lifetime boundary must remain usable"));
}
let after = store.get_task_snapshot(&task_id).unwrap().unwrap();
let expected = final_input_task_with_keys(&working, &["b".to_owned()]);
assert_ne!(after.generation(), current.generation());
assert_eq!(
serde_json::to_value(after.task()).unwrap(),
serde_json::to_value(&expected).unwrap()
);
assert_eq!(
serde_json::to_value(store.latest_notification(&task_id).unwrap()).unwrap(),
serde_json::to_value(final_task_notification(&expected)).unwrap(),
);
let state = store.state.lock().unwrap();
let history = state.input_key_history.get(&task_id).unwrap();
let mut expected_keys: BTreeSet<String> = earlier_keys.into_iter().collect();
expected_keys.insert("b".to_owned());
assert_eq!(history.keys, expected_keys);
if count_limit {
assert_eq!(history.keys.len(), MAX_IN_MEMORY_FINAL_TASK_INPUT_KEYS);
} else {
assert_eq!(history.key_bytes, MAX_IN_MEMORY_FINAL_TASK_INPUT_KEY_BYTES);
}
}
#[test]
fn task_02_b_input_key_count_at_limit_is_accepted() {
assert_input_key_history_limit(true, false);
}
#[test]
fn task_02_b_input_key_count_over_limit_is_atomic() {
assert_input_key_history_limit(true, true);
}
#[test]
fn task_02_b_input_key_bytes_at_limit_are_accepted() {
assert_input_key_history_limit(false, false);
}
#[test]
fn task_02_b_input_key_bytes_over_limit_is_atomic() {
assert_input_key_history_limit(false, true);
}
fn assert_input_key_create_byte_limit(over_limit: bool) {
let store = InMemoryFinalTaskStore::default();
let working = final_working_task_without_ttl("task-key-create-bound");
let task_id = working.base().task_id.clone();
let before = input_key_store_snapshot(&store, &task_id);
let key = "a".repeat(MAX_IN_MEMORY_FINAL_TASK_INPUT_KEY_BYTES + usize::from(over_limit));
let input = final_input_task_with_keys(&working, &[key]);
let result = store.create_task(input.clone(), final_task_notification(&input));
if over_limit {
let error = result.expect_err("oversized initial history must not create a task");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
assert_eq!(
error.message,
"Task input request key history byte limit exceeded"
);
assert_eq!(input_key_store_snapshot(&store, &task_id), before);
let admitted = final_input_task_with_keys(
&working,
&["a".repeat(MAX_IN_MEMORY_FINAL_TASK_INPUT_KEY_BYTES)],
);
store
.create_task(admitted.clone(), final_task_notification(&admitted))
.unwrap();
} else {
result.expect("exact-boundary initial input history remains creatable");
}
let state = store.state.lock().unwrap();
let history = state.input_key_history.get(&task_id).unwrap();
assert_eq!(history.keys.len(), 1);
assert_eq!(history.key_bytes, MAX_IN_MEMORY_FINAL_TASK_INPUT_KEY_BYTES);
assert_eq!(state.tasks.len(), 1);
assert_eq!(state.generations.get(&task_id), Some(&1));
assert_eq!(
history.keys.first().unwrap().len(),
MAX_IN_MEMORY_FINAL_TASK_INPUT_KEY_BYTES
);
}
#[test]
fn task_02_b_input_key_create_at_byte_limit_is_retained() {
assert_input_key_create_byte_limit(false);
}
#[test]
fn task_02_b_input_key_create_over_byte_limit_is_atomic() {
assert_input_key_create_byte_limit(true);
}
#[test]
fn task_02_b_input_key_history_survives_until_expiry() {
let (store, now) = in_memory_store_with_test_clock(1);
let working = final_working_task_with_ttl("task-key-history-expiry", 1_000);
let task_id = working.base().task_id.clone();
let keys = ["roots-a".to_owned(), "roots-b".to_owned()];
let input = final_input_task_with_keys(&working, &keys);
store
.create_task(input.clone(), final_task_notification(&input))
.unwrap();
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let first_response =
serde_json::from_value(serde_json::json!({"roots-a": {"roots": []}})).unwrap();
runtime.update_task(&task_id, &first_response).unwrap();
let before = input_key_store_snapshot(&store, &task_id);
let partial = store.get_task_snapshot(&task_id).unwrap().unwrap();
let error = store
.replace_task_if_current(&partial, input.clone(), final_task_notification(&input))
.expect_err("a satisfied key cannot reappear beside a still-outstanding key");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
assert_eq!(error.message, "Task input request keys cannot be reused");
assert_eq!(input_key_store_snapshot(&store, &task_id), before);
drop(runtime);
let recovered = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let second_response =
serde_json::from_value(serde_json::json!({"roots-b": {"roots": []}})).unwrap();
recovered.update_task(&task_id, &second_response).unwrap();
let current = store.get_task_snapshot(&task_id).unwrap().unwrap();
let before = input_key_store_snapshot(&store, &task_id);
let error = store
.replace_task_if_current(¤t, input.clone(), final_task_notification(&input))
.expect_err("recreating a runtime cannot reset the retained store key history");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
assert_eq!(error.message, "Task input request keys cannot be reused");
assert_eq!(input_key_store_snapshot(&store, &task_id), before);
let mut terminal_base = working.base().clone();
terminal_base.status = FinalTaskStatus::Cancelled;
let terminal = FinalTask::Cancelled(terminal_base);
store
.replace_task(terminal.clone(), final_task_notification(&terminal))
.unwrap();
let history_keys = || {
store
.state
.lock()
.unwrap()
.input_key_history
.get(&task_id)
.map(|history| history.keys.clone())
};
assert_eq!(history_keys(), Some(keys.iter().cloned().collect()));
{
let mut clock = now.lock().unwrap();
*clock = clock.checked_add(StdDuration::from_millis(999)).unwrap();
}
assert!(store.get_task(&task_id).unwrap().is_some());
assert!(
history_keys().is_some(),
"terminal history stays until actual expiry"
);
{
let mut clock = now.lock().unwrap();
*clock = clock.checked_add(StdDuration::from_millis(1)).unwrap();
}
assert!(store.get_task(&task_id).unwrap().is_none());
assert!(history_keys().is_none());
store
.create_task(input.clone(), final_task_notification(&input))
.unwrap();
assert_eq!(history_keys(), Some(keys.into_iter().collect()));
}
fn assert_outstanding_input_schema_identity(change_schema: bool) {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let working = final_working_task_without_ttl("task-outstanding-schema-identity");
let task_id = working.base().task_id.clone();
let mut base = working.base().clone();
base.status = FinalTaskStatus::InputRequired;
let requests = serde_json::from_value(serde_json::json!({
"roots": {"method": "roots/list"},
"form": {
"method": "elicitation/create",
"params": {
"mode": "form",
"message": "Enter the answer",
"requestedSchema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"]
}
}
}
}))
.expect("both initial embedded requests are admitted");
let initial = FinalTask::InputRequired {
base,
input_requests: requests,
};
store
.create_task(initial.clone(), final_task_notification(&initial))
.unwrap();
let first = serde_json::from_value(serde_json::json!({"roots": {"roots": []}})).unwrap();
runtime.update_task(&task_id, &first).unwrap();
let current = store.get_task_snapshot(&task_id).unwrap().unwrap();
let original = current.task().clone();
let before = input_key_store_snapshot(&store, &task_id);
assert_eq!(
before["accepted_inputs"],
serde_json::to_value(&first).unwrap()
);
let mut proposed = original.clone();
let FinalTask::InputRequired { input_requests, .. } = &mut proposed else {
panic!("partial acceptance must retain the outstanding form request");
};
assert_eq!(input_requests.len(), 1);
if change_schema {
let descriptor = input_requests.get_mut("form").unwrap();
let mut value = serde_json::to_value(&*descriptor).unwrap();
value["params"]["requestedSchema"]["properties"]["answer"]["type"] =
serde_json::json!("integer");
*descriptor = serde_json::from_value(value).expect("changed schema remains admitted");
}
let result = store.replace_task_and_append_input_if_current(
¤t,
proposed.clone(),
final_task_notification(&proposed),
BTreeMap::new(),
);
if change_schema {
let error = result.expect_err("same-kind schema changes must not retarget a key");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
assert_eq!(error.message, "Task input request keys cannot be reused");
assert_eq!(input_key_store_snapshot(&store, &task_id), before);
assert!(
store
.replace_task_and_append_input_if_current(
¤t,
original.clone(),
final_task_notification(&original),
BTreeMap::new(),
)
.expect("the identical descriptor can retry under the same generation")
.is_some()
);
} else {
assert!(
result
.expect("an identical outstanding descriptor remains valid")
.is_some()
);
}
let after = store.get_task_snapshot(&task_id).unwrap().unwrap();
assert_ne!(after.generation(), current.generation());
assert_eq!(
serde_json::to_value(after.task()).unwrap(),
serde_json::to_value(&original).unwrap()
);
assert_eq!(
serde_json::to_value(store.latest_notification(&task_id).unwrap()).unwrap(),
serde_json::to_value(final_task_notification(&original)).unwrap()
);
assert_eq!(
input_key_store_snapshot(&store, &task_id)["accepted_inputs"],
before["accepted_inputs"]
);
let second = serde_json::from_value(serde_json::json!({
"form": {"action": "accept", "content": {"answer": "original answer"}}
}))
.unwrap();
runtime.update_task(&task_id, &second).unwrap();
let completed_input = store.get_task_snapshot(&task_id).unwrap().unwrap();
assert!(matches!(completed_input.task(), FinalTask::Working(_)));
let retained = input_key_store_snapshot(&store, &task_id);
assert_eq!(
retained["accepted_inputs"],
serde_json::json!({
"roots": {"roots": []},
"form": {"action": "accept", "content": {"answer": "original answer"}}
})
);
assert_eq!(
retained["history"],
serde_json::json!([["form", "roots"], 9])
);
assert_eq!(
retained["notification"],
serde_json::to_value(final_task_notification(completed_input.task())).unwrap()
);
}
#[test]
fn task_02_b_input_key_history_rejects_changing_kind_of_outstanding_key() {
assert_outstanding_input_schema_identity(true);
let store = InMemoryFinalTaskStore::default();
let working = final_working_task_without_ttl("task-key-history-change-kind");
let task_id = working.base().task_id.clone();
let mut base = working.base().clone();
base.status = FinalTaskStatus::InputRequired;
let mut roots_requests = FinalTaskInputRequests::new();
roots_requests.insert(
"query".to_owned(),
serde_json::from_value(serde_json::json!({"method": "roots/list"}))
.expect("typed roots input request"),
);
let initial_input = FinalTask::InputRequired {
base: base.clone(),
input_requests: roots_requests,
};
store
.create_task(
initial_input.clone(),
final_task_notification(&initial_input),
)
.expect("initial input task creates");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("read task snapshot")
.expect("snapshot exists");
let before_store_state = input_key_store_snapshot(&store, &task_id);
let mut sampling_requests = FinalTaskInputRequests::new();
sampling_requests.insert(
"query".to_owned(),
serde_json::from_value(serde_json::json!({
"method": "sampling/createMessage",
"params": {
"messages": [],
"maxTokens": 16
}
}))
.expect("typed sampling input request"),
);
let sampling_replacement = FinalTask::InputRequired {
base: transition_final_task_base(
base.clone(),
FinalTaskStatus::InputRequired,
Some("attempt kind change to sampling".to_owned()),
)
.expect("base transition is valid"),
input_requests: sampling_requests,
};
let sampling_error = store
.replace_task_if_current(
&snapshot,
sampling_replacement.clone(),
final_task_notification(&sampling_replacement),
)
.expect_err(
"reusing an outstanding input key with a different response kind must fail",
);
assert_eq!(
sampling_error.code,
fastmcp_core::McpErrorCode::InvalidParams
);
assert_eq!(
sampling_error.message,
"Task input request keys cannot be reused"
);
assert_eq!(
input_key_store_snapshot(&store, &task_id),
before_store_state
);
let current_snapshot = store
.get_task_snapshot(&task_id)
.expect("read current snapshot")
.expect("task remains present");
assert_eq!(current_snapshot.generation(), snapshot.generation());
assert_eq!(
serde_json::to_value(current_snapshot.task()).expect("serialize current task"),
serde_json::to_value(snapshot.task()).expect("serialize snapshot task")
);
assert_eq!(
serde_json::to_value(store.latest_notification(&task_id).unwrap())
.expect("serialize notification"),
serde_json::to_value(final_task_notification(snapshot.task()))
.expect("serialize expected notification")
);
let mut elicitation_requests = FinalTaskInputRequests::new();
elicitation_requests.insert(
"query".to_owned(),
serde_json::from_value(serde_json::json!({
"method": "elicitation/create",
"params": {
"mode": "url",
"message": "please visit url",
"url": "https://example.com/auth"
}
}))
.expect("typed elicitation input request"),
);
let elicitation_replacement = FinalTask::InputRequired {
base: transition_final_task_base(
base,
FinalTaskStatus::InputRequired,
Some("attempt kind change to elicitation".to_owned()),
)
.expect("base transition is valid"),
input_requests: elicitation_requests,
};
let elicitation_error = store
.replace_task_if_current(
&snapshot,
elicitation_replacement.clone(),
final_task_notification(&elicitation_replacement),
)
.expect_err(
"reusing an outstanding input key with elicitation response kind must fail",
);
assert_eq!(
elicitation_error.code,
fastmcp_core::McpErrorCode::InvalidParams
);
assert_eq!(
elicitation_error.message,
"Task input request keys cannot be reused"
);
assert_eq!(
input_key_store_snapshot(&store, &task_id),
before_store_state
);
}
#[test]
fn task_02_b_input_key_history_retains_unmodified_outstanding_keys() {
assert_outstanding_input_schema_identity(false);
let store = InMemoryFinalTaskStore::default();
let working = final_working_task_without_ttl("task-key-history-retain-matching");
let task_id = working.base().task_id.clone();
let mut base = working.base().clone();
base.status = FinalTaskStatus::InputRequired;
let mut initial_requests = FinalTaskInputRequests::new();
initial_requests.insert(
"query".to_owned(),
serde_json::from_value(serde_json::json!({"method": "roots/list"}))
.expect("typed roots input request"),
);
let initial_input = FinalTask::InputRequired {
base: base.clone(),
input_requests: initial_requests.clone(),
};
store
.create_task(
initial_input.clone(),
final_task_notification(&initial_input),
)
.expect("initial input task creates");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("snapshot is readable")
.expect("snapshot exists");
{
let state = store.state.lock().unwrap();
let history = state
.input_key_history
.get(&task_id)
.expect("history exists");
assert_eq!(history.keys.len(), 1);
assert!(history.keys.contains("query"));
assert_eq!(history.key_bytes, "query".len());
}
let replacement_same = FinalTask::InputRequired {
base: transition_final_task_base(
base.clone(),
FinalTaskStatus::InputRequired,
Some("retaining query with matching kind".to_owned()),
)
.expect("base transition is valid"),
input_requests: initial_requests.clone(),
};
assert!(
store
.replace_task_if_current(
&snapshot,
replacement_same.clone(),
final_task_notification(&replacement_same),
)
.expect("replace with matching kind succeeds")
);
let snapshot_2 = store
.get_task_snapshot(&task_id)
.expect("snapshot is readable")
.expect("snapshot exists");
assert_ne!(snapshot_2.generation(), snapshot.generation());
assert_eq!(
serde_json::to_value(snapshot_2.task()).unwrap(),
serde_json::to_value(&replacement_same).unwrap()
);
assert_eq!(
serde_json::to_value(store.latest_notification(&task_id).unwrap()).unwrap(),
serde_json::to_value(final_task_notification(&replacement_same)).unwrap()
);
{
let state = store.state.lock().unwrap();
let history = state
.input_key_history
.get(&task_id)
.expect("history exists");
assert_eq!(history.keys.len(), 1);
assert_eq!(history.key_bytes, "query".len());
}
let mut updated_requests = initial_requests;
updated_requests.insert(
"query-2".to_owned(),
serde_json::from_value(serde_json::json!({"method": "roots/list"}))
.expect("typed roots input request"),
);
let replacement_extended = FinalTask::InputRequired {
base: transition_final_task_base(
base,
FinalTaskStatus::InputRequired,
Some("retaining query and adding query-2".to_owned()),
)
.expect("base transition is valid"),
input_requests: updated_requests,
};
assert!(
store
.replace_task_if_current(
&snapshot_2,
replacement_extended.clone(),
final_task_notification(&replacement_extended),
)
.expect("replace with matching kind and new key succeeds")
);
let snapshot_3 = store.get_task_snapshot(&task_id).unwrap().unwrap();
assert_ne!(snapshot_3.generation(), snapshot_2.generation());
assert_eq!(
serde_json::to_value(snapshot_3.task()).unwrap(),
serde_json::to_value(&replacement_extended).unwrap()
);
assert_eq!(
serde_json::to_value(store.latest_notification(&task_id).unwrap()).unwrap(),
serde_json::to_value(final_task_notification(&replacement_extended)).unwrap()
);
let state = store.state.lock().unwrap();
let history = state
.input_key_history
.get(&task_id)
.expect("history exists");
assert_eq!(history.keys.len(), 2);
assert!(history.keys.contains("query"));
assert!(history.keys.contains("query-2"));
assert_eq!(history.key_bytes, "query".len() + "query-2".len());
}
#[test]
fn task_03_in_memory_store_rejects_stale_transition_after_terminal_commit() {
let store = InMemoryFinalTaskStore::default();
let working = final_working_task_without_ttl("task-atomic-transition");
let task_id = working.base().task_id.clone();
store
.create_task(working.clone(), final_task_notification(&working))
.expect("working task creates");
let working_snapshot = store
.get_task_snapshot(&task_id)
.expect("working task snapshot is readable")
.expect("working task snapshot is retained");
let mut cancelled_base = working.base().clone();
cancelled_base.status = FinalTaskStatus::Cancelled;
let cancelled = FinalTask::Cancelled(cancelled_base);
assert!(
store
.replace_task_if_current(
&working_snapshot,
cancelled.clone(),
final_task_notification(&cancelled),
)
.expect("terminal compare-and-replace is readable")
);
let mut input_required_base = working.base().clone();
input_required_base.status = FinalTaskStatus::InputRequired;
let stale_input_required = FinalTask::InputRequired {
base: input_required_base,
input_requests: final_roots_request(),
};
assert!(
!store
.replace_task_if_current(
&working_snapshot,
stale_input_required.clone(),
final_task_notification(&stale_input_required),
)
.expect("stale compare-and-replace is readable"),
"the stale working snapshot cannot overwrite a terminal transition"
);
assert!(matches!(
store
.get_task(&task_id)
.expect("terminal task lookup is readable"),
Some(FinalTask::Cancelled(_))
));
}
#[test]
fn task_03_final_task_snapshot_public_constructor_retains_opaque_generation() {
let task = final_working_task_without_ttl("task-public-snapshot");
let task_id = task.base().task_id.clone();
let snapshot = FinalTaskSnapshot::new(task, 41);
assert_eq!(snapshot.task().base().task_id, task_id);
assert_eq!(snapshot.generation(), 41);
}
#[test]
fn task_03_in_memory_store_generation_rejects_aba_replacement() {
let store = InMemoryFinalTaskStore::default();
let working = final_working_task_without_ttl("task-generation-aba");
let task_id = working.base().task_id.clone();
store
.create_task(working.clone(), final_task_notification(&working))
.expect("working task creates");
let initial_snapshot = store
.get_task_snapshot(&task_id)
.expect("initial snapshot is readable")
.expect("working task is retained");
assert!(
store
.replace_task_if_current(
&initial_snapshot,
working.clone(),
final_task_notification(&working),
)
.expect("same-value replacement is accepted for the current generation")
);
assert!(
!store
.replace_task_if_current(
&initial_snapshot,
working.clone(),
final_task_notification(&working),
)
.expect("stale same-value replacement is readable"),
"only the store generation changes, so a reused wire value cannot pass CAS"
);
}
#[test]
fn task_03_in_memory_store_allows_status_update_but_rejects_ttl_drift_without_losing_recovery()
{
const TTL_MS: u64 = 60_000;
let (store, now) = in_memory_store_with_test_clock(1);
let expiring = final_working_task_with_ttl("task-replacement-expiry", TTL_MS);
let task_id = expiring.base().task_id.clone();
store
.create_task_with_work(
expiring.clone(),
final_task_notification(&expiring),
final_test_work_descriptor(),
)
.expect("expiring task and its initial supervisor handoff create atomically");
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(TTL_MS - 1))
.expect("test clock reaches one millisecond before creation expiry");
drop(clock);
let snapshot_before_update = store
.get_task_snapshot(&task_id)
.expect("pre-deadline task snapshot is readable")
.expect("pre-deadline task remains retained");
let FinalTask::Working(mut updated_base) = expiring.clone() else {
unreachable!("the fixture is a working task");
};
updated_base.status_message = Some("still working".to_owned());
let updated = FinalTask::Working(updated_base);
assert!(
store
.replace_task_if_current(
&snapshot_before_update,
updated.clone(),
final_task_notification(&updated),
)
.expect("an otherwise identical working update is accepted"),
"changing only the mutable status message preserves the creation retention contract"
);
let snapshot_before_rejection = store
.get_task_snapshot(&task_id)
.expect("post-update task snapshot is readable")
.expect("post-update task remains retained");
let task_before_rejection = serde_json::to_value(snapshot_before_rejection.task())
.expect("serialize retained task before TTL rejection");
let notification_before_rejection = serde_json::to_value(
store
.latest_notification(&task_id)
.expect("post-update notification remains retained"),
)
.expect("serialize retained notification before TTL rejection");
let FinalTask::Working(mut ttl_drift_base) = updated else {
unreachable!("the accepted update remains a working task");
};
ttl_drift_base.ttl_ms = None;
let ttl_drift = FinalTask::Working(ttl_drift_base);
let error = store
.replace_task_if_current(
&snapshot_before_rejection,
ttl_drift.clone(),
final_task_notification(&ttl_drift),
)
.expect_err("changing only ttlMs must not rewrite durable retention");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
assert!(error.message.contains("ttlMs"));
let snapshot_after_rejection = store
.get_task_snapshot(&task_id)
.expect("task snapshot remains readable after TTL rejection")
.expect("TTL rejection cannot remove the retained task");
assert_eq!(
serde_json::to_value(snapshot_after_rejection.task())
.expect("serialize retained task after TTL rejection"),
task_before_rejection,
"the near-identical rejected transition preserves the durable task"
);
assert_eq!(
snapshot_after_rejection.generation(),
snapshot_before_rejection.generation(),
"the rejected TTL drift cannot advance the durable generation"
);
assert_eq!(
serde_json::to_value(
store
.latest_notification(&task_id)
.expect("notification remains retained after TTL rejection"),
)
.expect("serialize retained notification after TTL rejection"),
notification_before_rejection,
"the rejected TTL drift cannot replace the durable notification"
);
let recovered = test_next_initial_work(&store)
.expect("initial supervisor handoff remains recoverable")
.expect("retention rejection cannot erase initial task work");
assert_eq!(recovered.task_id(), &task_id);
assert_eq!(
store
.work_descriptor_if_current(&recovered)
.expect("recovered snapshot exposes its durable work descriptor")
.expect("initial work retains its descriptor"),
final_test_work_descriptor()
);
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(1))
.expect("test clock reaches the original creation deadline");
drop(clock);
assert!(
store
.get_task(&task_id)
.expect("expired task lookup is readable")
.is_none(),
"the accepted status update and rejected TTL drift retain the original creation deadline"
);
}
#[test]
fn task_03_in_memory_runtime_reclaims_expired_task_before_capacity_check_positive() {
let store = Arc::new(
InMemoryFinalTaskStore::new(1).expect("one retained task is a valid bounded store"),
);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let first = create_final_task_state_fixture(&runtime, None);
let first_id = first.task.base().task_id.clone();
{
let mut state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.expires_at.insert(
first_id.clone(),
std::time::Instant::now()
.checked_sub(std::time::Duration::from_millis(1))
.expect("a just-created monotonic instant can be moved back one millisecond"),
);
}
let second = create_final_task_state_fixture(&runtime, None);
let second_id = second.task.base().task_id.clone();
assert_eq!(store.task_count(), 1);
assert!(
store
.get_task(&first_id)
.expect("expired task lookup is readable")
.is_none(),
"reclamation removes the expired task"
);
assert!(store.latest_notification(&first_id).is_none());
assert!(
store
.get_task(&second_id)
.expect("replacement task lookup is readable")
.is_some()
);
}
#[test]
fn task_03_in_memory_store_rejects_one_field_notification_task_id_mismatch() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let first_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let second_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let first_task = store
.get_task(&first_id)
.expect("first task reads")
.expect("first task remains retained");
let first_notification = store
.latest_notification(&first_id)
.expect("first notification remains retained");
let mut mismatched_notification = first_notification.clone();
let FinalTask::Working(base) = &mut mismatched_notification.params.task else {
panic!("created task notification must begin in the working state");
};
base.task_id = second_id;
let first_task_before = serde_json::to_value(&first_task).expect("serialize first task");
let first_notification_before =
serde_json::to_value(&first_notification).expect("serialize first notification");
let error = store
.replace_task(first_task, mismatched_notification)
.expect_err("only the notification task ID differs from the accepted replacement");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
assert_eq!(
store.task_count(),
2,
"rejection preserves both retained tasks"
);
let first_task_after = store
.get_task(&first_id)
.expect("first task remains readable after rejection")
.expect("rejection cannot remove the retained task");
assert_eq!(
serde_json::to_value(first_task_after).expect("serialize post-rejection task"),
first_task_before,
"mismatched notification cannot replace the retained task"
);
let first_notification_after = store
.latest_notification(&first_id)
.expect("rejection cannot remove the retained notification");
assert_eq!(
serde_json::to_value(first_notification_after)
.expect("serialize post-rejection notification"),
first_notification_before,
"mismatched notification cannot replace the retained notification"
);
}
#[test]
fn task_03_in_memory_store_rejects_same_id_notification_base_drift_without_mutation() {
let store = InMemoryFinalTaskStore::default();
let task = final_working_task_without_ttl("task-notification-base-drift");
let task_id = task.base().task_id.clone();
let notification = final_task_notification(&task);
store
.create_task(task.clone(), notification.clone())
.expect("matching task and notification create");
let snapshot_before = store
.get_task_snapshot(&task_id)
.expect("stored task snapshot is readable")
.expect("created task is retained");
let task_before = serde_json::to_value(snapshot_before.task())
.expect("serialize retained task before rejection");
let notification_before = serde_json::to_value(¬ification)
.expect("serialize retained notification before rejection");
let mut drifted_notification = notification.clone();
let FinalTask::Working(base) = &mut drifted_notification.params.task else {
panic!("baseline notification contains the working task");
};
base.status_message = Some("only the notification task base drifted".to_owned());
let error = store
.replace_task_if_current(&snapshot_before, task, drifted_notification)
.expect_err("same-ID notification base drift must be rejected");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
let snapshot_after = store
.get_task_snapshot(&task_id)
.expect("stored task snapshot remains readable")
.expect("rejection preserves the retained task");
assert_eq!(
serde_json::to_value(snapshot_after.task())
.expect("serialize retained task after rejection"),
task_before,
"rejection preserves the retained task"
);
assert_eq!(
snapshot_after.generation(),
snapshot_before.generation(),
"rejection preserves the compare-and-swap generation"
);
assert_eq!(
serde_json::to_value(
store
.latest_notification(&task_id)
.expect("rejection preserves the retained notification"),
)
.expect("serialize retained notification after rejection"),
notification_before,
"rejection preserves the retained notification"
);
}
#[test]
fn task_03_final_durable_runtime_positive() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let delivered_after_durable_commit = Arc::new(AtomicBool::new(false));
let runtime = final_task_runtime(
Arc::clone(&store),
Arc::clone(&delivered_after_durable_commit),
);
let service_runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install caller-owned service runner before task advertisement");
let service_cx = Cx::for_testing();
let _running_service = enter_task_service_runner(service_runner, &service_cx);
let created = runtime
.create_task_with_work(final_test_work_descriptor(), Some("accepted".to_owned()))
.expect("durable create before wire reply");
let task_id = created.task.base().task_id.clone();
assert!(matches!(created.task, FinalTask::Working(_)));
assert_eq!(store.task_count(), 1, "create result retains one task");
assert!(
delivered_after_durable_commit.load(AtomicOrdering::SeqCst),
"typed notification delivery runs only after the store has accepted the task"
);
let created_notification = store
.latest_notification(&task_id)
.expect("durable create records its typed notification");
let notification_wire =
serde_json::to_value(created_notification).expect("encode task notification");
assert_eq!(notification_wire["method"], "notifications/tasks");
assert_eq!(notification_wire["params"]["taskId"], task_id.as_str());
assert_eq!(
runtime
.get_task(&task_id)
.expect("get newly created task")
.task
.base()
.task_id,
task_id
);
runtime
.require_input(
&task_id,
final_roots_request(),
Some("awaiting roots".to_owned()),
)
.expect("working task accepts typed roots request");
let input_responses: FinalTaskInputResponses = serde_json::from_value(serde_json::json!({
"roots": {"roots": []}
}))
.expect("typed roots response");
let update = runtime
.update_task(&task_id, &input_responses)
.expect("matching typed input response updates task");
assert_eq!(
serde_json::to_value(update).expect("encode empty update acknowledgement")["resultType"],
"complete"
);
assert!(matches!(
runtime
.get_task(&task_id)
.expect("get task after update")
.task,
FinalTask::Working(_)
));
let cancel = runtime
.cancel_task(&task_id)
.expect("durably record cancellation intent");
assert_eq!(
serde_json::to_value(cancel).expect("encode empty cancel acknowledgement")["resultType"],
"complete"
);
assert!(matches!(
runtime
.get_task(&task_id)
.expect("read durable terminal cancellation")
.task,
FinalTask::Cancelled(_)
));
assert!(
!runtime
.is_cancellation_requested(&task_id)
.expect("terminal task has no outstanding cooperative intent"),
"only an elected handoff retains cooperative cancellation intent"
);
assert!(
store.latest_notification(&task_id).is_some(),
"the bounded store retains the terminal typed notification"
);
}
#[test]
fn task_03_final_notification_emitters_deliver_after_durable_mutation() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let delivered = Arc::new(AtomicBool::new(false));
let delivered_by_emitter = Arc::clone(&delivered);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(move |_| {
delivered_by_emitter.store(true, AtomicOrdering::SeqCst);
}),
);
let task = final_working_task_without_ttl("task-emitter-positive");
let task_id = task.base().task_id.clone();
runtime
.persist_new_with_work(task, final_test_work_descriptor())
.expect("a non-panicking emitter preserves successful durable mutation");
assert!(
delivered.load(AtomicOrdering::SeqCst),
"the installed emitter receives the post-commit notification"
);
assert!(
store
.get_task(&task_id)
.expect("read task after notification delivery")
.is_some(),
"the notification observes a task that was already durable"
);
}
#[test]
fn task_03_final_panicking_emitter_preserves_accepted_create_and_continues() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let continued = Arc::new(AtomicBool::new(false));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| panic!("planted final task notification emitter panic")),
);
let continued_by_second_emitter = Arc::clone(&continued);
runtime.add_notification_emitter(Arc::new(move |_| {
continued_by_second_emitter.store(true, AtomicOrdering::SeqCst);
}));
let runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install a ready service before public task creation");
let service_cx = Cx::for_testing();
let _running_service = enter_task_service_runner(runner, &service_cx);
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
runtime.create_task_with_work(final_test_work_descriptor(), None)
}));
let created = result
.expect("an emitter panic is contained after the durable write")
.expect("a post-commit emitter panic cannot turn accepted creation into an error");
let task_id = created.task.base().task_id.clone();
assert!(
continued.load(AtomicOrdering::SeqCst),
"a later emitter still receives the same durable notification after one panic"
);
assert!(
store
.get_task(&task_id)
.expect("read task after contained emitter panic")
.is_some(),
"the durable task mutation survives the contained emitter panic"
);
}
#[test]
fn task_03_final_tasks_update_commits_unexpired_precommit_snapshot() {
let (store, runtime, task_id) =
final_update_precommit_fixture(StdDuration::from_millis(999));
let mut parameters = final_task_method_parameters(&task_id);
parameters["inputResponses"] = serde_json::json!({"roots": {"roots": []}});
let response = dispatch_final_tasks_update(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
parameters,
)
.expect("an unexpired precommit snapshot admits tasks/update");
assert_eq!(response["resultType"], "complete");
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
state.tasks.get(&task_id),
Some(FinalTask::Working(_))
));
assert!(state.generations.contains_key(&task_id));
assert!(
state
.accepted_inputs
.get(&task_id)
.is_some_and(|responses| { responses.contains_key("roots") })
);
assert!(matches!(
state.latest_notifications.get(&task_id),
Some(notification) if matches!(¬ification.params.task, FinalTask::Working(_))
));
}
#[test]
fn task_03_final_tasks_update_rejects_expired_precommit_snapshot() {
let (store, runtime, task_id) =
final_update_precommit_fixture(StdDuration::from_millis(1_000));
let mut parameters = final_task_method_parameters(&task_id);
parameters["inputResponses"] = serde_json::json!({"roots": {"roots": []}});
let error = dispatch_final_tasks_update(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
parameters,
)
.expect_err("an expired precommit snapshot cannot mutate through tasks/update");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
assert_eq!(error.data, None);
assert_eq!(
error.message,
"Task state changed before the transition could be recorded"
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(!state.tasks.contains_key(&task_id));
assert!(!state.generations.contains_key(&task_id));
assert!(!state.accepted_inputs.contains_key(&task_id));
assert!(!state.latest_notifications.contains_key(&task_id));
}
#[test]
fn task_03_final_replacement_cas_preserves_live_snapshot() {
for (variant, target_name) in [
(0_u8, "task-replacement-live-direct"),
(1_u8, "task-replacement-live-append"),
(2_u8, "task-replacement-live-clear"),
] {
let (store, now) = in_memory_store_with_test_clock(2);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, None)
.expect("replacement fixture retention is valid"),
Arc::new(|_| {}),
);
let target = final_working_task_with_ttl(target_name, 1_000);
let target_id = target.base().task_id.clone();
store
.create_task_with_work(
target.clone(),
final_task_notification(&target),
final_test_work_descriptor(),
)
.expect("replacement target fixture is retained");
runtime
.require_input(&target_id, final_roots_request(), None)
.expect("replacement target enters input_required");
let sibling = final_working_task_with_ttl("task-replacement-live-sibling", 2_000);
let sibling_id = sibling.base().task_id.clone();
store
.create_task_with_work(
sibling.clone(),
final_task_notification(&sibling),
final_test_work_descriptor(),
)
.expect("replacement sibling fixture is retained");
runtime
.require_input(&sibling_id, final_roots_request(), None)
.expect("replacement sibling enters input_required");
let target_snapshot = store
.get_task_snapshot(&target_id)
.expect("live replacement snapshot is readable")
.expect("live replacement task is retained");
let sibling_snapshot = store
.get_task_snapshot(&sibling_id)
.expect("live replacement sibling snapshot is readable")
.expect("live replacement sibling is retained");
let sibling_notification = store
.latest_notification(&sibling_id)
.expect("live replacement sibling notification is retained");
let sibling_input = {
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.accepted_inputs.get(&sibling_id).cloned()
};
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
StdDuration::from_millis(999);
let FinalTask::InputRequired {
base,
input_requests,
} = target_snapshot.task()
else {
unreachable!("replacement fixture awaits input");
};
let replacement = FinalTask::InputRequired {
base: transition_final_task_base(
base.clone(),
FinalTaskStatus::InputRequired,
Some("replacement".to_owned()),
)
.expect("live replacement transition is structurally valid"),
input_requests: input_requests.clone(),
};
let expected_generation = {
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state
.next_generation
.checked_add(1)
.expect("live replacement generation remains representable")
};
let replacement_value =
serde_json::to_value(&replacement).expect("encode live replacement task");
let replacement_notification = final_task_notification(&replacement);
let replacement_notification_value = serde_json::to_value(&replacement_notification)
.expect("encode live replacement notification");
let append_input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({
"roots": {"roots": []}
}))
.expect("live replacement input is valid");
let committed = match variant {
0 => FinalTaskStore::replace_task_if_current(
&*store,
&target_snapshot,
replacement,
replacement_notification,
)
.expect("live direct replacement CAS is readable"),
1 => FinalTaskStore::replace_task_and_append_input_if_current(
&*store,
&target_snapshot,
replacement,
replacement_notification,
append_input_responses.clone(),
)
.expect("live append replacement CAS is readable")
.is_some(),
2 => FinalTaskStore::replace_task_and_clear_input_if_current(
&*store,
&target_snapshot,
replacement,
replacement_notification,
)
.expect("live clear replacement CAS is readable")
.is_some(),
_ => unreachable!("all replacement variants are covered"),
};
assert!(committed, "an unexpired replacement CAS commits");
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
state.generations.get(&target_id),
Some(&expected_generation)
);
assert_eq!(
serde_json::to_value(state.tasks.get(&target_id))
.expect("encode committed replacement task"),
replacement_value
);
assert_eq!(
serde_json::to_value(state.latest_notifications.get(&target_id))
.expect("encode committed replacement notification"),
replacement_notification_value
);
match variant {
0 | 2 => assert!(!state.accepted_inputs.contains_key(&target_id)),
1 => assert_eq!(
serde_json::to_value(state.accepted_inputs.get(&target_id))
.expect("encode appended replacement input"),
serde_json::to_value(&append_input_responses)
.expect("encode expected replacement input")
),
_ => unreachable!("all replacement variants are covered"),
}
assert_eq!(
serde_json::to_value(state.tasks.get(&sibling_id))
.expect("encode retained replacement sibling task"),
serde_json::to_value(sibling_snapshot.task())
.expect("encode pre-expiry replacement sibling task")
);
assert_eq!(
state.generations.get(&sibling_id),
Some(&sibling_snapshot.generation())
);
assert_eq!(
serde_json::to_value(state.latest_notifications.get(&sibling_id))
.expect("encode retained replacement sibling notification"),
serde_json::to_value(&sibling_notification)
.expect("encode pre-expiry replacement sibling notification")
);
assert_eq!(
state.accepted_inputs.get(&sibling_id),
sibling_input.as_ref()
);
}
}
#[test]
fn task_03_final_replacement_cas_rejects_expired_snapshot() {
for (variant, target_name) in [
(0_u8, "task-replacement-expired-direct"),
(1_u8, "task-replacement-expired-append"),
(2_u8, "task-replacement-expired-clear"),
] {
let (store, now) = in_memory_store_with_test_clock(2);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, None)
.expect("replacement fixture retention is valid"),
Arc::new(|_| {}),
);
let target = final_working_task_with_ttl(target_name, 1_000);
let target_id = target.base().task_id.clone();
store
.create_task_with_work(
target.clone(),
final_task_notification(&target),
final_test_work_descriptor(),
)
.expect("replacement target fixture is retained");
runtime
.require_input(&target_id, final_roots_request(), None)
.expect("replacement target enters input_required");
let sibling = final_working_task_with_ttl("task-replacement-expired-sibling", 2_000);
let sibling_id = sibling.base().task_id.clone();
store
.create_task_with_work(
sibling.clone(),
final_task_notification(&sibling),
final_test_work_descriptor(),
)
.expect("replacement sibling fixture is retained");
runtime
.require_input(&sibling_id, final_roots_request(), None)
.expect("replacement sibling enters input_required");
let target_snapshot = store
.get_task_snapshot(&target_id)
.expect("expired replacement snapshot is readable before expiry")
.expect("expired replacement task is retained before expiry");
let sibling_snapshot = store
.get_task_snapshot(&sibling_id)
.expect("expired replacement sibling snapshot is readable")
.expect("expired replacement sibling is retained");
let sibling_notification = store
.latest_notification(&sibling_id)
.expect("expired replacement sibling notification is retained");
let sibling_input = {
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.accepted_inputs.get(&sibling_id).cloned()
};
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) += StdDuration::from_secs(1);
let FinalTask::InputRequired {
base,
input_requests,
} = target_snapshot.task()
else {
unreachable!("replacement fixture awaits input");
};
let replacement = FinalTask::InputRequired {
base: transition_final_task_base(
base.clone(),
FinalTaskStatus::InputRequired,
Some("replacement".to_owned()),
)
.expect("expired replacement transition is structurally valid"),
input_requests: input_requests.clone(),
};
let next_generation_before = {
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.next_generation
};
let replacement_notification = final_task_notification(&replacement);
let rejected = match variant {
0 => !FinalTaskStore::replace_task_if_current(
&*store,
&target_snapshot,
replacement,
replacement_notification,
)
.expect("expired direct replacement CAS is readable"),
1 => {
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({
"roots": {"roots": []}
}))
.expect("expired replacement input is valid");
FinalTaskStore::replace_task_and_append_input_if_current(
&*store,
&target_snapshot,
replacement,
replacement_notification,
input_responses,
)
.expect("expired append replacement CAS is readable")
.is_none()
}
2 => FinalTaskStore::replace_task_and_clear_input_if_current(
&*store,
&target_snapshot,
replacement,
replacement_notification,
)
.expect("expired clear replacement CAS is readable")
.is_none(),
_ => unreachable!("all replacement variants are covered"),
};
assert!(rejected, "retention expiry fences every replacement CAS");
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
state.next_generation, next_generation_before,
"expired replacement reclamation does not allocate a replacement generation"
);
assert!(!state.tasks.contains_key(&target_id));
assert!(!state.generations.contains_key(&target_id));
assert!(!state.accepted_inputs.contains_key(&target_id));
assert!(!state.latest_notifications.contains_key(&target_id));
assert_eq!(
serde_json::to_value(state.tasks.get(&sibling_id))
.expect("encode retained replacement sibling task"),
serde_json::to_value(sibling_snapshot.task())
.expect("encode pre-expiry replacement sibling task")
);
assert_eq!(
state.generations.get(&sibling_id),
Some(&sibling_snapshot.generation())
);
assert_eq!(
serde_json::to_value(state.latest_notifications.get(&sibling_id))
.expect("encode retained replacement sibling notification"),
serde_json::to_value(&sibling_notification)
.expect("encode pre-expiry replacement sibling notification")
);
assert_eq!(
state.accepted_inputs.get(&sibling_id),
sibling_input.as_ref()
);
}
}
#[test]
fn task_03_final_update_non_panicking_emitter_preserves_committed_state() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let primary_delivered = Arc::new(AtomicBool::new(false));
let primary_delivered_by_emitter = Arc::clone(&primary_delivered);
let runtime = FinalTaskRuntime::new(
store.clone() as Arc<dyn FinalTaskStore>,
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(move |_| {
primary_delivered_by_emitter.store(true, AtomicOrdering::SeqCst);
}),
);
let continued = Arc::new(AtomicBool::new(false));
let continued_by_second_emitter = Arc::clone(&continued);
runtime.add_notification_emitter(Arc::new(move |_| {
continued_by_second_emitter.store(true, AtomicOrdering::SeqCst);
}));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
runtime
.require_input(&task_id, final_roots_request(), None)
.expect("task enters input_required before the update RPC");
primary_delivered.store(false, AtomicOrdering::SeqCst);
continued.store(false, AtomicOrdering::SeqCst);
let mut parameters = final_task_method_parameters(&task_id);
parameters["inputResponses"] = serde_json::json!({"roots": {"roots": []}});
let response = dispatch_final_tasks_update(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
parameters,
)
.expect("a delivered post-commit notification preserves the update RPC success");
assert_eq!(response["resultType"], "complete");
assert!(
primary_delivered.load(AtomicOrdering::SeqCst),
"the first emitter receives the committed update notification"
);
assert!(
continued.load(AtomicOrdering::SeqCst),
"the later emitter receives the same committed update notification"
);
assert!(matches!(
runtime
.get_task(&task_id)
.expect("read task after successful update RPC")
.task,
FinalTask::Working(_)
));
assert!(matches!(
store
.latest_notification(&task_id)
.expect("read retained update notification")
.params
.task,
FinalTask::Working(_)
));
}
#[test]
fn task_03_final_update_panicking_emitter_preserves_committed_state_and_replay_safety() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = FinalTaskRuntime::new(
store.clone() as Arc<dyn FinalTaskStore>,
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| panic!("planted final task notification emitter panic")),
);
let continued = Arc::new(AtomicBool::new(false));
let continued_by_second_emitter = Arc::clone(&continued);
runtime.add_notification_emitter(Arc::new(move |_| {
continued_by_second_emitter.store(true, AtomicOrdering::SeqCst);
}));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
runtime
.require_input(&task_id, final_roots_request(), None)
.expect("task enters input_required despite prior delivery degradation");
continued.store(false, AtomicOrdering::SeqCst);
let mut parameters = final_task_method_parameters(&task_id);
parameters["inputResponses"] = serde_json::json!({"roots": {"roots": []}});
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
dispatch_final_tasks_update(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
parameters.clone(),
)
}));
let response = result
.expect("a panicking emitter is contained after the durable update")
.expect("delivery degradation cannot turn the committed update RPC into an error");
assert_eq!(response["resultType"], "complete");
assert!(
continued.load(AtomicOrdering::SeqCst),
"a later emitter still receives the committed update notification"
);
assert!(matches!(
runtime
.get_task(&task_id)
.expect("read task after contained emitter panic")
.task,
FinalTask::Working(_)
));
assert!(matches!(
store
.latest_notification(&task_id)
.expect("read retained notification after contained emitter panic")
.params
.task,
FinalTask::Working(_)
));
let generation_after_commit = store
.get_task_snapshot(&task_id)
.expect("read durable generation after committed update")
.expect("committed update retains its task")
.generation();
let replay = dispatch_final_tasks_update(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
parameters,
)
.expect("the retry after delivery degradation is acknowledged as a replay");
assert_eq!(replay["resultType"], "complete");
assert_eq!(
store
.get_task_snapshot(&task_id)
.expect("read durable generation after replay")
.expect("replay retains its task")
.generation(),
generation_after_commit,
"replaying the accepted update cannot create a second durable transition"
);
}
#[test]
fn task_03_final_get_dispatch_requires_official_task_id_and_metadata() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let response = dispatch_final_tasks_get(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
final_task_method_parameters(&task_id),
)
.expect("official final tasks/get parameters are admitted");
assert_eq!(response["resultType"], serde_json::json!("complete"));
assert_eq!(response["taskId"], serde_json::json!(task_id));
assert!(
dispatch_final_tasks_get(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
serde_json::json!({
"id": task_id.clone(),
"_meta": {
"io.modelcontextprotocol/protocolVersion": FINAL_PROTOCOL_VERSION,
"io.modelcontextprotocol/clientCapabilities": {}
}
}),
)
.is_err(),
"changing only taskId to the legacy id field fails final strict decoding"
);
}
#[test]
fn task_03_final_cancel_dispatch_requires_official_task_id_and_metadata() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let missing_capabilities = serde_json::json!({
"taskId": task_id.clone(),
"_meta": {
"io.modelcontextprotocol/protocolVersion": FINAL_PROTOCOL_VERSION
}
});
assert!(
dispatch_final_tasks_cancel(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
missing_capabilities
)
.is_err(),
"changing only the required modern metadata fails final tasks/cancel admission"
);
let response = dispatch_final_tasks_cancel(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
final_task_method_parameters(&task_id),
)
.expect("official final tasks/cancel parameters are admitted");
assert_eq!(response["resultType"], "complete");
assert!(matches!(
runtime
.get_task(&task_id)
.expect("official final cancellation persists terminal state")
.task,
FinalTask::Cancelled(_)
));
assert!(
!runtime
.is_cancellation_requested(&task_id)
.expect("terminal task has no outstanding cooperative intent")
);
}
#[test]
fn task_03_final_update_dispatch_requires_exact_metadata() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
runtime
.require_input(&task_id, final_roots_request(), None)
.expect("task awaits one typed final input response");
let input_responses = serde_json::json!({"roots": {"roots": []}});
let missing_metadata = serde_json::json!({
"taskId": task_id.clone(),
"inputResponses": input_responses.clone(),
});
assert!(
dispatch_final_tasks_update(
&runtime,
&McpContext::new(Cx::for_testing(), 1),
missing_metadata
)
.is_err(),
"changing only the final request metadata rejects tasks/update before mutation"
);
assert!(matches!(
runtime
.get_task(&task_id)
.expect("read task after rejected update metadata")
.task,
FinalTask::InputRequired { .. }
));
let mut admitted = final_task_method_parameters(&task_id);
admitted["inputResponses"] = input_responses;
let response =
dispatch_final_tasks_update(&runtime, &McpContext::new(Cx::for_testing(), 1), admitted)
.expect("exact final metadata admits tasks/update");
assert_eq!(response["resultType"], "complete");
assert!(matches!(
runtime
.get_task(&task_id)
.expect("read task after admitted final update")
.task,
FinalTask::Working(_)
));
}
#[test]
fn task_03_final_accepted_input_reaches_resumed_supervisor() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let mut requests = final_roots_request();
requests.insert(
"workspace-roots".to_owned(),
serde_json::from_value(serde_json::json!({"method": "roots/list"}))
.expect("typed second roots input request"),
);
runtime
.require_input(
&task_id,
requests,
Some("awaiting both roots responses".to_owned()),
)
.expect("working task requests two typed inputs");
let first: FinalTaskInputResponses = serde_json::from_value(serde_json::json!({
"roots": {"roots": [{"uri": "file:///first"}]}
}))
.expect("typed first roots response");
runtime
.update_task(&task_id, &first)
.expect("accept first matching input response");
assert!(
runtime
.take_accepted_input(&task_id)
.expect("read input handoff while task remains input_required")
.is_none(),
"a supervisor cannot resume until every outstanding input is satisfied"
);
let second: FinalTaskInputResponses = serde_json::from_value(serde_json::json!({
"workspace-roots": {"roots": [{"uri": "file:///second"}]}
}))
.expect("typed second roots response");
runtime
.update_task(&task_id, &second)
.expect("accept final matching input response");
let resumed_generation = store
.get_task_snapshot(&task_id)
.expect("read resumed task generation")
.expect("resumed task remains retained")
.generation();
let accepted = runtime
.take_accepted_input(&task_id)
.expect("resumed task exposes one supervisor handoff")
.expect("all accepted input values are retained for the resumed worker");
assert_eq!(accepted.task_id(), &task_id);
assert_eq!(accepted.generation(), resumed_generation);
assert_eq!(accepted.input_responses().get("roots"), first.get("roots"));
assert_eq!(
accepted.input_responses().get("workspace-roots"),
second.get("workspace-roots")
);
assert!(matches!(
runtime
.get_task(&task_id)
.expect("resumed task remains readable")
.task,
FinalTask::Working(_)
));
assert!(
runtime
.take_accepted_input(&task_id)
.expect("second handoff read is valid")
.is_none(),
"the supervisor handoff is one-shot and cannot replay accepted input"
);
}
#[test]
fn task_03_final_new_input_cycle_clears_unconsumed_prior_handoff() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let mut first_requests = FinalTaskInputRequests::new();
first_requests.insert(
"first-roots".to_owned(),
serde_json::from_value(serde_json::json!({"method": "roots/list"}))
.expect("typed first-cycle roots request"),
);
runtime
.require_input(&task_id, first_requests, None)
.expect("enter first input cycle");
let first_responses: FinalTaskInputResponses = serde_json::from_value(serde_json::json!({
"first-roots": {"roots": [{"uri": "file:///first-cycle"}]}
}))
.expect("typed first-cycle response");
runtime
.update_task(&task_id, &first_responses)
.expect("complete first input cycle without consuming its handoff");
let mut second_requests = FinalTaskInputRequests::new();
second_requests.insert(
"second-roots".to_owned(),
serde_json::from_value(serde_json::json!({"method": "roots/list"}))
.expect("typed second-cycle roots request"),
);
runtime
.require_input(&task_id, second_requests, None)
.expect("enter second input cycle and clear the first handoff");
let second_responses: FinalTaskInputResponses = serde_json::from_value(serde_json::json!({
"second-roots": {"roots": [{"uri": "file:///second-cycle"}]}
}))
.expect("typed second-cycle response");
runtime
.update_task(&task_id, &second_responses)
.expect("complete second input cycle");
let accepted = runtime
.take_accepted_input(&task_id)
.expect("read second-cycle supervisor handoff")
.expect("second cycle retains its accepted response");
assert_eq!(accepted.input_responses(), &second_responses);
assert!(
!accepted.input_responses().contains_key("first-roots"),
"starting a new cycle atomically removes unconsumed prior-cycle input"
);
}
#[test]
fn task_03_final_terminal_transition_fences_stale_input_take() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
runtime
.require_input(&task_id, final_roots_request(), None)
.expect("task requests roots before terminal race");
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed roots response");
runtime
.update_task(&task_id, &input_responses)
.expect("accepted input returns task to working");
let stale_working = store
.get_task_snapshot(&task_id)
.expect("read pre-terminal generation")
.expect("working task remains retained");
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal tool result");
runtime
.complete_task(&task_id, result, None)
.expect("terminal transition wins before stale supervisor take");
let terminal = store
.get_task_snapshot(&task_id)
.expect("read terminal generation")
.expect("terminal task remains retained");
assert_ne!(terminal.generation(), stale_working.generation());
assert!(matches!(terminal.task(), FinalTask::Completed { .. }));
assert!(
test_take_input(&store, &stale_working)
.expect("stale generation take fails closed")
.is_none(),
"a stale working generation cannot consume after a terminal winner"
);
assert!(
runtime
.take_accepted_input(&task_id)
.expect("terminal task has no supervisor handoff")
.is_none(),
"the terminal transition atomically clears previously accepted input"
);
}
#[test]
fn task_03_final_new_runtime_recovers_unconsumed_input_handoff() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let first_runtime =
final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&first_runtime, None)
.task
.base()
.task_id
.clone();
first_runtime
.require_input(&task_id, final_roots_request(), None)
.expect("task requests input before runtime replacement");
let input_responses: FinalTaskInputResponses = serde_json::from_value(
serde_json::json!({"roots": {"roots": [{"uri": "file:///recovered"}]}}),
)
.expect("typed retained roots response");
first_runtime
.update_task(&task_id, &input_responses)
.expect("store commits task state and input together");
drop(first_runtime);
let recovered_runtime =
final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let accepted = recovered_runtime
.take_accepted_input(&task_id)
.expect("new runtime reads the store-owned handoff")
.expect("unconsumed accepted input survives runtime replacement");
assert_eq!(accepted.input_responses(), &input_responses);
assert!(
recovered_runtime
.take_accepted_input(&task_id)
.expect("second recovered take is valid")
.is_none(),
"the recovered handoff remains one-shot"
);
}
#[test]
fn task_03_final_recovery_continues_after_lost_candidate_cas() {
let inner = Arc::new(InMemoryFinalTaskStore::default());
let setup_runtime =
final_task_runtime(Arc::clone(&inner), Arc::new(AtomicBool::new(false)));
for (task_id, uri) in [
("task-lost-cas-first", "file:///lost-candidate"),
("task-lost-cas-second", "file:///surviving-candidate"),
] {
let task = final_working_task_without_ttl(task_id);
let task_id = task.base().task_id.clone();
inner
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.expect("durably create recoverable task work");
setup_runtime
.require_input(&task_id, final_roots_request(), None)
.expect("task requests roots before recovery race");
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": [{"uri": uri}]}}))
.expect("typed retained roots response");
setup_runtime
.update_task(&task_id, &input_responses)
.expect("task retains accepted input for recovery");
}
let recovery_store = Arc::new(LoseFirstAcceptedRecoveryCandidateStore::new(Arc::clone(
&inner,
)));
let recovery_runtime = FinalTaskRuntime::new(
recovery_store,
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| {}),
);
let recovered = recovery_runtime
.recover_accepted_input()
.expect("recovery retries after a lost candidate compare-and-take")
.expect("a second accepted handoff remains recoverable after the first CAS loss");
let recovered_wire = serde_json::to_value(recovered.input_responses())
.expect("serialize recovered accepted input");
assert_eq!(
recovered_wire["roots"]["roots"][0]["uri"],
serde_json::json!("file:///surviving-candidate"),
"only the first candidate loses its CAS; recovery continues to the next durable handoff"
);
}
#[test]
fn task_03_final_creation_requires_ready_service_and_recovers_initial_work() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let work_descriptor = FinalTaskWorkDescriptor::new(serde_json::json!({
"handler": "initial-work",
"payload": {"request": 7}
}))
.expect("non-null application work descriptor is valid");
assert!(
runtime.ensure_task_service_ready().is_err(),
"the read-only readiness probe fails closed before installation"
);
assert!(
runtime
.create_task_with_work(work_descriptor.clone(), None)
.is_err(),
"changing only the absent service authority fails task creation before advertisement"
);
assert!(
runtime.create_task(None).is_err(),
"bare task creation cannot omit the durable application work descriptor"
);
let started = Arc::new(Mutex::new(Vec::new()));
let runner = runtime
.install_task_service(
1,
Arc::new(RecordingInitialFinalTaskSupervisor {
started: Arc::clone(&started),
}),
)
.expect("install caller-owned task service runner");
assert!(
runtime
.create_task_with_work(work_descriptor.clone(), None)
.is_err(),
"installing a runner without entering run does not authorize task advertisement"
);
assert!(
runtime.ensure_task_service_ready().is_err(),
"the probe remains false until the runner has entered"
);
let readiness_cx = Cx::for_testing();
let running_service = enter_task_service_runner(runner, &readiness_cx);
runtime
.ensure_task_service_ready()
.expect("an entered live runner owns the probe generation");
let created = runtime
.create_task_with_work(work_descriptor.clone(), Some("accepted".to_owned()))
.expect("entered service permits durable task creation and advertisement");
let task_id = created.task.base().task_id.clone();
drop(running_service);
assert!(
runtime.ensure_task_service_ready().is_err(),
"dropping the entered runner revokes its readiness generation"
);
assert!(
runtime
.create_task_with_work(final_test_work_descriptor(), None)
.is_err(),
"dropping the entered runner revokes creation readiness immediately"
);
let runner = runtime
.install_task_service(
1,
Arc::new(RecordingInitialFinalTaskSupervisor {
started: Arc::clone(&started),
}),
)
.expect("a dropped runner releases service readiness for recovery");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
let cx = Cx::for_testing();
application_runtime
.block_on(runner.run(&cx))
.expect("initial durable work is recovered by the caller-owned supervisor");
assert!(
runtime.ensure_task_service_ready().is_err(),
"a runner that exits from run revokes the readiness probe"
);
assert!(
runtime
.create_task_with_work(final_test_work_descriptor(), None)
.is_err(),
"a runner that exits from run revokes creation readiness"
);
assert_eq!(
started
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_slice(),
&[(task_id, work_descriptor)],
"the supervisor receives the exact descriptor bound before task advertisement"
);
}
#[test]
fn task_03_final_public_creation_holds_readiness_through_durable_commit() {
let inner = Arc::new(InMemoryFinalTaskStore::default());
let store = Arc::new(ReadinessLeaseProbeFinalTaskStore::new(inner));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, None).expect("valid final task policy"),
Arc::new(|_| {}),
);
store.observe_service_signal(Arc::clone(&runtime.service_signal));
let runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install caller-owned task service runner");
let service_cx = Cx::for_testing();
let running_service = enter_task_service_runner(runner, &service_cx);
runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("an entered runner accepts a durably recoverable task");
assert!(
store.observed_ready_lease.load(AtomicOrdering::SeqCst),
"the durable create commit holds the exact ready service generation"
);
drop(running_service);
}
#[test]
fn task_03_final_public_creation_without_entered_runner_never_reaches_durable_commit() {
let inner = Arc::new(InMemoryFinalTaskStore::default());
let store = Arc::new(ReadinessLeaseProbeFinalTaskStore::new(inner));
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::new(60_000, None).expect("valid final task policy"),
Arc::new(|_| {}),
);
store.observe_service_signal(Arc::clone(&runtime.service_signal));
let _runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("installing alone does not enter a task service runner");
assert!(
runtime
.create_task_with_work(final_test_work_descriptor(), None)
.is_err(),
"changing only entered runner state rejects public task creation"
);
assert!(
!store.observed_ready_lease.load(AtomicOrdering::SeqCst),
"a rejected creation cannot call the durable store"
);
}
#[test]
fn task_03_final_cancelled_before_entry_never_publishes_readiness_or_creates() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let advertised = Arc::new(AtomicBool::new(false));
let runtime = final_task_runtime(Arc::clone(&store), Arc::clone(&advertised));
let runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("installing a runner reserves, but does not establish, readiness");
let cx = Cx::for_testing();
cx.cancel_with(CancelKind::User, None);
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
assert!(
application_runtime.block_on(runner.run(&cx)).is_err(),
"an already-cancelled runner stops at its entry checkpoint"
);
assert!(
runtime.ensure_task_service_ready().is_err(),
"a cancelled-before-entry runner has zero readiness authority"
);
assert!(
runtime
.create_task_with_work(final_test_work_descriptor(), None)
.is_err(),
"zero readiness prevents task creation before durable mutation"
);
assert!(
!advertised.load(AtomicOrdering::SeqCst),
"the failed creation attempt emits no durable task advertisement"
);
}
#[test]
fn task_03_final_run_service_error_revokes_readiness_and_retries_exact_initial_work() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task = final_working_task_without_ttl("task-run-service-error-initial");
let task_id = task.base().task_id.clone();
let work_descriptor = FinalTaskWorkDescriptor::new(serde_json::json!({
"handler": "run-service-error",
"payload": {"initial": true}
}))
.expect("non-null initial work descriptor is valid");
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durable before the service starts");
let action = Arc::new(AtomicUsize::new(RUN_SERVICE_SUPERVISOR_FAIL));
let mut runner = runtime
.install_task_service(
1,
Arc::new(SwitchableRunServiceSupervisor {
action: Arc::clone(&action),
}),
)
.expect("install retained task service runner");
let failed_cx = Cx::for_testing();
assert!(matches!(
poll_retained_task_service(&mut runner, &failed_cx),
std::task::Poll::Ready(Err(_))
));
assert!(
!runtime.is_task_service_ready(),
"a supervisor error must revoke the entered retained service readiness"
);
assert_exact_initial_work_is_recoverable(&store, &task_id, &work_descriptor);
action.store(RUN_SERVICE_SUPERVISOR_COMPLETE, AtomicOrdering::SeqCst);
let retry_cx = Cx::for_testing();
assert!(matches!(
poll_retained_task_service(&mut runner, &retry_cx),
std::task::Poll::Ready(Ok(()))
));
assert!(
!runtime.is_task_service_ready(),
"the caller-owned retry exit must revoke readiness again"
);
assert!(matches!(
store
.get_task(&task_id)
.expect("retried initial task is readable"),
Some(FinalTask::Completed { .. })
));
assert!(
test_next_initial_work(&store)
.expect("completed initial recovery scan is valid")
.is_none(),
"the successful retry consumes the exact recovered initial handoff"
);
}
#[test]
fn task_03_final_run_service_cancellation_revokes_readiness_and_retries_exact_input() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses = serde_json::from_value(
serde_json::json!({"roots": {"roots": [{"uri": "file:///run-service-cancel"}]}}),
)
.expect("typed accepted input is valid");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let action = Arc::new(AtomicUsize::new(RUN_SERVICE_SUPERVISOR_PENDING));
let mut runner = runtime
.install_task_service(
1,
Arc::new(SwitchableRunServiceSupervisor {
action: Arc::clone(&action),
}),
)
.expect("install retained task service runner");
let cancelled_cx = Cx::for_testing();
let mut service = Box::pin(runner.run_service(&cancelled_cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
Future::poll(service.as_mut(), &mut context),
std::task::Poll::Pending
));
assert!(
runtime.is_task_service_ready(),
"an entered retained service is the only live readiness authority"
);
cancelled_cx.cancel_with(CancelKind::User, None);
assert!(matches!(
Future::poll(service.as_mut(), &mut context),
std::task::Poll::Ready(Ok(()))
));
drop(service);
assert!(
!runtime.is_task_service_ready(),
"caller-context cancellation must revoke retained service readiness"
);
assert_exact_accepted_input_is_recoverable(&store, &task_id, &input_responses);
action.store(RUN_SERVICE_SUPERVISOR_COMPLETE, AtomicOrdering::SeqCst);
let retry_cx = Cx::for_testing();
assert!(matches!(
poll_retained_task_service(&mut runner, &retry_cx),
std::task::Poll::Ready(Ok(()))
));
assert!(matches!(
store
.get_task(&task_id)
.expect("retried accepted-input task is readable"),
Some(FinalTask::Completed { .. })
));
assert!(
test_next_accepted_input(&store)
.expect("completed accepted-input recovery scan is valid")
.is_none(),
"the successful retry consumes the exact recovered accepted-input handoff"
);
}
#[test]
fn task_03_final_run_service_drop_revokes_readiness_and_retries_exact_initial_work() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task = final_working_task_without_ttl("task-run-service-drop-initial");
let task_id = task.base().task_id.clone();
let work_descriptor = FinalTaskWorkDescriptor::new(serde_json::json!({
"handler": "run-service-drop",
"payload": {"initial": true}
}))
.expect("non-null initial work descriptor is valid");
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durable before the service starts");
let action = Arc::new(AtomicUsize::new(RUN_SERVICE_SUPERVISOR_PENDING));
let mut runner = runtime
.install_task_service(
1,
Arc::new(SwitchableRunServiceSupervisor {
action: Arc::clone(&action),
}),
)
.expect("install retained task service runner");
let service_cx = Cx::for_testing();
let mut service = Box::pin(runner.run_service(&service_cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
Future::poll(service.as_mut(), &mut context),
std::task::Poll::Pending
));
assert!(runtime.is_task_service_ready());
drop(service);
assert!(
!runtime.is_task_service_ready(),
"dropping a retained service future must revoke readiness"
);
assert_exact_initial_work_is_recoverable(&store, &task_id, &work_descriptor);
action.store(RUN_SERVICE_SUPERVISOR_COMPLETE, AtomicOrdering::SeqCst);
let retry_cx = Cx::for_testing();
assert!(matches!(
poll_retained_task_service(&mut runner, &retry_cx),
std::task::Poll::Ready(Ok(()))
));
assert!(matches!(
store
.get_task(&task_id)
.expect("retried dropped-service task is readable"),
Some(FinalTask::Completed { .. })
));
}
#[test]
fn task_03_final_run_service_non_concurrent_runner_boundary_is_live() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let action = Arc::new(AtomicUsize::new(RUN_SERVICE_SUPERVISOR_PENDING));
let mut runner = runtime
.install_task_service(
1,
Arc::new(SwitchableRunServiceSupervisor {
action: Arc::clone(&action),
}),
)
.expect("install the only retained service runner");
let service_cx = Cx::for_testing();
let mut service = Box::pin(runner.run_service(&service_cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
Future::poll(service.as_mut(), &mut context),
std::task::Poll::Pending
));
assert!(runtime.is_task_service_ready());
assert!(
runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.is_err(),
"a live retained service generation rejects a second service owner"
);
drop(service);
assert!(!runtime.is_task_service_ready());
}
#[test]
fn task_03_final_run_service_reentry_republishes_readiness_on_the_same_runner() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let mut runner = runtime
.install_task_service(1, Arc::new(PendingFinalTaskSupervisor))
.expect("install retained task service runner");
let first_cx = Cx::for_testing();
let mut first_service = Box::pin(runner.run_service(&first_cx));
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
Future::poll(first_service.as_mut(), &mut context),
std::task::Poll::Pending
));
assert!(
runtime.is_task_service_ready(),
"the first retained service entry publishes readiness"
);
drop(first_service);
assert!(
!runtime.is_task_service_ready(),
"dropping the first service future revokes its readiness lease"
);
let second_cx = Cx::for_testing();
let mut second_service = Box::pin(runner.run_service(&second_cx));
assert!(matches!(
Future::poll(second_service.as_mut(), &mut context),
std::task::Poll::Pending
));
assert!(
runtime.is_task_service_ready(),
"the same retained runner can re-enter and publish a new readiness lease"
);
drop(second_service);
assert!(
!runtime.is_task_service_ready(),
"the re-entered service future also revokes readiness on drop"
);
}
#[test]
fn task_03_final_initial_handoff_error_restores_exact_work_descriptor() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let work_descriptor = FinalTaskWorkDescriptor::new(serde_json::json!({
"handler": "initial-error-recovery",
"payload": {"request": 8}
}))
.expect("non-null application work descriptor is valid");
let runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install caller-owned service runner before task advertisement");
let readiness_cx = Cx::for_testing();
let running_service = enter_task_service_runner(runner, &readiness_cx);
let task_id = runtime
.create_task_with_work(work_descriptor.clone(), None)
.expect("entered service permits initial durable work")
.task
.base()
.task_id
.clone();
drop(running_service);
let runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("a dropped runner releases readiness for initial-work recovery");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
assert!(
application_runtime
.block_on(runner.run(&Cx::for_testing()))
.is_err(),
"the application supervisor error remains visible after restoring initial work"
);
let restored = runtime
.recover_initial_work()
.expect("initial recovery scan reads the restored descriptor")
.expect("a failed initial supervisor handoff is restored");
assert_eq!(restored.task_id(), &task_id);
assert_eq!(restored.work_descriptor(), &work_descriptor);
}
#[test]
fn task_03_in_memory_initial_work_lease_expires_and_recovers_exact_descriptor() {
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_without_ttl("task-initial-lease-expiry");
let task_id = task.base().task_id.clone();
let work_descriptor = final_test_work_descriptor();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained with its task");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial task snapshot is readable")
.expect("initial task snapshot is retained");
assert_eq!(
test_take_initial_work(&store, &snapshot).expect("initial handoff lease is claimable"),
Some(work_descriptor.clone())
);
assert!(
test_next_initial_work(&store)
.expect("leased initial work scan is readable")
.is_none(),
"a live recovery lease prevents concurrent delivery"
);
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(IN_MEMORY_FINAL_TASK_HANDOFF_LEASE)
.expect("fixed handoff lease fits the monotonic test clock");
drop(clock);
let recovered = test_next_initial_work(&store)
.expect("expired initial-work lease scan is readable")
.expect("an expired claim makes the durable initial work recoverable");
assert_eq!(recovered.task().base().task_id, task_id);
assert_eq!(
test_take_initial_work(&store, &recovered)
.expect("expired lease permits a new initial claim"),
Some(work_descriptor),
"lease expiry changes only recovery eligibility, not the durable descriptor"
);
}
#[test]
fn task_03_in_memory_initial_work_lease_one_millisecond_before_expiry_blocks_recovery() {
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_without_ttl("task-initial-lease-pre-expiry");
let task_id = task.base().task_id.clone();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.expect("initial work is durably retained with its task");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial task snapshot is readable")
.expect("initial task snapshot is retained");
assert!(
test_take_initial_work(&store, &snapshot)
.expect("initial handoff lease is claimable")
.is_some()
);
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE
.checked_sub(StdDuration::from_millis(1))
.expect("fixed handoff lease exceeds one millisecond"),
)
.expect("pre-expiry handoff lease fits the monotonic test clock");
drop(clock);
assert!(
test_next_initial_work(&store)
.expect("pre-expiry initial-work scan is readable")
.is_none(),
"changing only the final millisecond keeps the live recovery lease exclusive"
);
}
#[test]
fn task_03_in_memory_resumed_input_lease_expires_and_recovers_exact_payload() {
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses = serde_json::from_value(
serde_json::json!({"roots": {"roots": [{"uri": "file:///lease-expiry"}]}}),
)
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let claimed = store
.get_task_snapshot(&task_id)
.expect("accepted-input task snapshot is readable")
.expect("accepted-input task snapshot is retained");
assert_eq!(
test_take_input(&store, &claimed).expect("accepted input claim is valid"),
Some(input_responses.clone())
);
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(IN_MEMORY_FINAL_TASK_HANDOFF_LEASE)
.expect("fixed handoff lease fits the monotonic test clock");
drop(clock);
let recovered = test_next_accepted_input(&store)
.expect("expired accepted-input lease scan is readable")
.expect("an expired accepted-input claim becomes recoverable");
assert_ne!(recovered.generation(), claimed.generation());
assert_eq!(
test_take_input(&store, &recovered).expect("expired input lease permits a new claim"),
Some(input_responses),
"lease expiry changes only recovery eligibility, not the retained input"
);
}
#[test]
fn task_03_in_memory_resumed_input_lease_one_millisecond_before_expiry_blocks_recovery() {
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses);
let claimed = store
.get_task_snapshot(&task_id)
.expect("accepted-input task snapshot is readable")
.expect("accepted-input task snapshot is retained");
assert!(
test_take_input(&store, &claimed)
.expect("accepted input claim is valid")
.is_some()
);
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE
.checked_sub(StdDuration::from_millis(1))
.expect("fixed handoff lease exceeds one millisecond"),
)
.expect("pre-expiry handoff lease fits the monotonic test clock");
drop(clock);
assert!(
test_next_accepted_input(&store)
.expect("pre-expiry accepted-input scan is readable")
.is_none(),
"changing only the final millisecond keeps the accepted-input claim exclusive"
);
}
#[test]
fn task_03_final_raw_input_claim_is_guarded_while_owned_claim_delivers_once() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let snapshot = store
.get_task_snapshot(&task_id)
.expect("accepted-input snapshot is readable")
.expect("accepted-input task remains retained");
assert!(
FinalTaskStore::take_input_if_current(&*store, &snapshot).is_err(),
"the legacy raw store claim is fail-closed without an execution owner"
);
assert_eq!(
FinalTaskStore::take_input_for_owner_if_current(&*store, &snapshot, "guarded-owner",)
.expect("owned handoff claim is valid"),
Some(input_responses),
"changing only the owner guard makes the durable input available exactly once"
);
}
#[test]
fn task_03_final_elected_dispatch_renewal_preserves_exclusive_ownership() {
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_without_ttl("task-elected-dispatch-renewal");
let task_id = task.base().task_id.clone();
let work_descriptor = final_test_work_descriptor();
let owner_id = "renewing-owner";
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained with its task");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial task snapshot is readable")
.expect("initial task snapshot is retained");
assert_eq!(
FinalTaskStore::take_initial_work_for_owner_if_current(&*store, &snapshot, owner_id)
.expect("initial handoff claim is valid"),
Some(work_descriptor)
);
let dispatch_fence = FinalTaskStore::begin_handoff_dispatch_for_owner_if_current(
&*store,
&task_id,
snapshot.generation(),
owner_id,
)
.expect("elected dispatch is valid")
.expect("claimed owner wins dispatch election");
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(
IN_MEMORY_FINAL_TASK_HANDOFF_LEASE
.checked_sub(StdDuration::from_millis(1))
.expect("fixed handoff lease exceeds one millisecond"),
)
.expect("pre-renewal handoff lease fits the monotonic test clock");
drop(clock);
assert!(
FinalTaskStore::renew_handoff_dispatch_if_current(
&*store,
&task_id,
snapshot.generation(),
owner_id,
dispatch_fence,
)
.expect("matching owner renews the durable dispatch lease"),
"renewing only the live owner keeps its fence current"
);
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(1))
.expect("renewed handoff lease fits the monotonic test clock");
drop(clock);
assert!(
FinalTaskStore::next_initial_work_snapshot_after(&*store, None)
.expect("renewed dispatch recovery scan is readable")
.is_none(),
"the matching renewal keeps an elected live supervisor exclusively fenced"
);
}
#[test]
fn task_03_in_memory_expired_dispatch_lease_fences_crashed_owner_completion() {
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_without_ttl("task-initial-lease-fence");
let task_id = task.base().task_id.clone();
let work_descriptor = final_test_work_descriptor();
let crashed_owner = "crashed-owner";
let recovery_owner = "recovery-owner";
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained with its task");
let abandoned = store
.get_task_snapshot(&task_id)
.expect("initial task snapshot is readable")
.expect("initial task snapshot is retained");
assert!(
FinalTaskStore::take_initial_work_for_owner_if_current(
&*store,
&abandoned,
crashed_owner,
)
.expect("initial handoff lease is claimable")
.is_some()
);
let crashed_fence = FinalTaskStore::begin_handoff_dispatch_for_owner_if_current(
&*store,
&task_id,
abandoned.generation(),
crashed_owner,
)
.expect("crashed owner dispatch election is valid")
.expect("claimed owner is elected before the simulated crash");
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(IN_MEMORY_FINAL_TASK_HANDOFF_LEASE)
.expect("fixed handoff lease fits the monotonic test clock");
drop(clock);
let replacement = test_next_initial_work(&store)
.expect("expired lease recovery scan is readable")
.expect("expired lease yields a newly fenced recovery candidate");
assert_ne!(replacement.generation(), abandoned.generation());
assert!(
FinalTaskStore::take_initial_work_for_owner_if_current(
&*store,
&replacement,
recovery_owner,
)
.expect("replacement recovery lease is claimable")
.is_some()
);
assert!(
!FinalTaskStore::finish_handoff_dispatch_for_owner_if_current(
&*store,
&task_id,
abandoned.generation(),
crashed_owner,
crashed_fence,
)
.expect("late completion observes its stale fenced owner"),
"an expired elected owner cannot complete or release a replacement recovery lease"
);
assert!(
test_next_initial_work(&store)
.expect("newer lease scan is readable")
.is_none(),
"the late stale restoration leaves the replacement lease exclusive"
);
}
#[test]
fn task_03_in_memory_expired_elected_cancellation_lease_retires_task() {
let (store, now) = in_memory_store_with_test_clock(1);
let task = final_working_task_without_ttl("task-expired-elected-cancellation");
let task_id = task.base().task_id.clone();
let owner_id = "cancelled-crashed-owner";
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.expect("initial work is durably retained with its task");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial task snapshot is readable")
.expect("initial task snapshot is retained");
assert!(
FinalTaskStore::take_initial_work_for_owner_if_current(&*store, &snapshot, owner_id)
.expect("initial handoff lease is claimable")
.is_some()
);
FinalTaskStore::begin_handoff_dispatch_for_owner_if_current(
&*store,
&task_id,
snapshot.generation(),
owner_id,
)
.expect("dispatch election is valid")
.expect("claimed owner wins dispatch election");
FinalTaskStore::request_cancellation(&*store, &task_id)
.expect("elected task records cooperative cancellation intent");
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(IN_MEMORY_FINAL_TASK_HANDOFF_LEASE)
.expect("fixed handoff lease fits the monotonic test clock");
drop(clock);
let retired = store
.get_task_snapshot(&task_id)
.expect("expiry reclamation leaves a readable terminal task")
.expect("unlimited-retention task remains stored after lease expiry");
assert!(matches!(retired.task(), FinalTask::Cancelled(_)));
assert!(
!store
.is_cancellation_requested(&task_id)
.expect("terminal retirement consumes cancellation intent")
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
!state.handoff_leases.contains_key(&task_id)
&& !state.initial_work.contains_key(&task_id)
&& !state.accepted_inputs.contains_key(&task_id),
"changing only cancellation from normal lease expiry retires rather than strands work"
);
}
#[test]
fn task_03_in_memory_cancellation_fences_claimed_initial_work() {
let store = InMemoryFinalTaskStore::default();
let task = final_working_task_without_ttl("task-initial-lease-cancel");
let task_id = task.base().task_id.clone();
let work_descriptor = final_test_work_descriptor();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained with its task");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial task snapshot is readable")
.expect("initial task snapshot is retained");
assert!(
test_take_initial_work(&store, &snapshot)
.expect("initial handoff lease is claimable")
.is_some()
);
let cancelled = FinalTask::Cancelled(
transition_terminal_final_task_base(
snapshot.task().base().clone(),
FinalTaskStatus::Cancelled,
None,
)
.expect("cancellation transition is valid"),
);
let after = store
.request_cancellation_and_clear_input_if_current(
&snapshot,
cancelled.clone(),
final_task_notification(&cancelled),
)
.expect("cancellation is atomically recorded against the claimed generation")
.expect("the claimed-but-unelected handoff is terminally cancelled");
assert!(matches!(after.task(), FinalTask::Cancelled(_)));
assert!(
!store
.is_cancellation_requested(&task_id)
.expect("terminal cancellation consumes cooperative intent")
);
assert!(
test_next_initial_work(&store)
.expect("cancelled initial-work scan is readable")
.is_none(),
"cancellation clears a claimed-but-not-dispatched initial handoff"
);
assert!(
!test_restore_initial_work(&store, &task_id, snapshot.generation(), work_descriptor)
.expect("stale claimed-work restoration is fenced by cancellation"),
"only cancellation differs from the retryable supervisor-error path"
);
}
#[test]
fn task_03_in_memory_cancellation_generation_exhaustion_preserves_claimed_work() {
let store = InMemoryFinalTaskStore::default();
let task = final_working_task_without_ttl("task-cancel-generation-exhaustion");
let task_id = task.base().task_id.clone();
let work_descriptor = final_test_work_descriptor();
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
work_descriptor.clone(),
)
.expect("initial work is durably retained with its task");
let snapshot = store
.get_task_snapshot(&task_id)
.expect("initial task snapshot is readable")
.expect("initial task snapshot is retained");
assert_eq!(
test_take_initial_work(&store, &snapshot).expect("initial handoff claim is valid"),
Some(work_descriptor.clone())
);
{
let mut state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.next_generation = u64::MAX;
}
let cancelled = FinalTask::Cancelled(
transition_terminal_final_task_base(
snapshot.task().base().clone(),
FinalTaskStatus::Cancelled,
None,
)
.expect("cancellation transition is valid"),
);
assert!(
store
.request_cancellation_and_clear_input_if_current(
&snapshot,
cancelled.clone(),
final_task_notification(&cancelled),
)
.is_err(),
"generation exhaustion rejects cancellation before durable mutation"
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
state.generations.get(&task_id),
Some(&snapshot.generation())
);
assert_eq!(state.initial_work.get(&task_id), Some(&work_descriptor));
assert!(
state.handoff_leases.get(&task_id).is_some_and(|lease| {
lease.generation == snapshot.generation()
&& lease.kind == InMemoryFinalTaskHandoffKind::Initial
&& !lease.dispatch_elected
}),
"the rejected cancellation leaves the original claim fence intact"
);
assert!(
!state.cancellation_requests.contains(&task_id),
"the rejected cancellation records no cooperative intent"
);
}
#[test]
fn task_03_final_in_memory_cancellation_reclaims_expired_preexpiry_snapshot() {
for (variant, target_name) in [
(0_u8, "task-cancel-expired-atomic"),
(1_u8, "task-cancel-expired-raw"),
(2_u8, "task-cancel-expired-direct"),
] {
let (store, now) = in_memory_store_with_test_clock(2);
let target_id = FinalTaskId::parse(target_name).unwrap();
let target = final_working_task_with_ttl(target_name, 1_000);
store
.create_task_with_work(
target.clone(),
final_task_notification(&target),
final_test_work_descriptor(),
)
.expect("finite-TTL cancellation fixture is retained");
let sibling_id = FinalTaskId::parse("task-cancel-expired-sibling").unwrap();
let sibling = final_working_task_with_ttl("task-cancel-expired-sibling", 2_000);
store
.create_task_with_work(
sibling.clone(),
final_task_notification(&sibling),
final_test_work_descriptor(),
)
.expect("live sibling cancellation fixture is retained");
let target_snapshot = store
.get_task_snapshot(&target_id)
.expect("expired cancellation snapshot is readable before expiry")
.expect("expired cancellation task is retained before expiry");
let sibling_snapshot = store
.get_task_snapshot(&sibling_id)
.expect("live sibling snapshot is readable before expiry")
.expect("live sibling is retained before expiry");
let sibling_notification = store
.latest_notification(&sibling_id)
.expect("live sibling notification is retained before expiry");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) += StdDuration::from_secs(1);
let cancelled = FinalTask::Cancelled(
transition_terminal_final_task_base(
target_snapshot.task().base().clone(),
FinalTaskStatus::Cancelled,
None,
)
.expect("expired cancellation transition is structurally valid"),
);
let mutation_rejected = match variant {
0 => {
let cancelled_notification = final_task_notification(&cancelled);
FinalTaskStore::request_cancellation_and_clear_input_if_current(
&*store,
&target_snapshot,
cancelled,
cancelled_notification,
)
.expect("expired atomic cancellation CAS is readable")
.is_none()
}
1 => !FinalTaskStore::request_cancellation_if_current(&*store, &target_snapshot)
.expect("expired raw cancellation CAS is readable"),
2 => {
let error = FinalTaskStore::request_cancellation(&*store, &target_id)
.expect_err("expired direct cancellation must report a missing task");
assert_eq!(error.message, "Task not found");
true
}
_ => unreachable!("all cancellation variants are covered"),
};
assert!(
mutation_rejected,
"retention expiry must fence each cancellation mutation variant"
);
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(!state.tasks.contains_key(&target_id));
assert!(!state.generations.contains_key(&target_id));
assert!(!state.cancellation_requests.contains(&target_id));
assert!(
!state.latest_notifications.contains_key(&target_id),
"expired target has no retained Cancelled notification"
);
assert!(!state.handoff_leases.contains_key(&target_id));
assert_eq!(
serde_json::to_value(state.tasks.get(&sibling_id))
.expect("encode retained sibling task"),
serde_json::to_value(sibling_snapshot.task())
.expect("encode pre-expiry sibling task"),
"reclaiming expired cancellation tasks leaves the sibling task unchanged"
);
assert_eq!(
state.generations.get(&sibling_id),
Some(&sibling_snapshot.generation())
);
assert_eq!(
serde_json::to_value(state.latest_notifications.get(&sibling_id))
.expect("encode retained sibling notification"),
serde_json::to_value(&sibling_notification)
.expect("encode pre-expiry sibling notification"),
"reclaiming expired cancellation tasks leaves the sibling notification unchanged"
);
assert!(!state.cancellation_requests.contains(&sibling_id));
}
}
#[test]
fn task_03_final_in_memory_cancellation_retains_live_preexpiry_snapshot() {
for (variant, target_name) in [
(0_u8, "task-cancel-live-atomic"),
(1_u8, "task-cancel-live-raw"),
(2_u8, "task-cancel-live-direct"),
] {
let (store, now) = in_memory_store_with_test_clock(2);
let target_id = FinalTaskId::parse(target_name).unwrap();
let target = final_working_task_with_ttl(target_name, 1_000);
store
.create_task_with_work(
target.clone(),
final_task_notification(&target),
final_test_work_descriptor(),
)
.expect("finite-TTL live-cancellation fixture is retained");
let sibling_id = FinalTaskId::parse("task-cancel-live-sibling").unwrap();
let sibling = final_working_task_with_ttl("task-cancel-live-sibling", 2_000);
store
.create_task_with_work(
sibling.clone(),
final_task_notification(&sibling),
final_test_work_descriptor(),
)
.expect("live sibling cancellation fixture is retained");
let target_snapshot = store
.get_task_snapshot(&target_id)
.expect("live cancellation snapshot is readable")
.expect("live cancellation task is retained");
let sibling_snapshot = store
.get_task_snapshot(&sibling_id)
.expect("live sibling snapshot is readable")
.expect("live sibling is retained");
let sibling_notification = store
.latest_notification(&sibling_id)
.expect("live sibling notification is retained");
*now.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) +=
StdDuration::from_millis(999);
let target_generation = {
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state
.next_generation
.checked_add(1)
.expect("live cancellation generation remains representable")
};
match variant {
0 => {
let cancelled = FinalTask::Cancelled(
transition_terminal_final_task_base(
target_snapshot.task().base().clone(),
FinalTaskStatus::Cancelled,
None,
)
.expect("live cancellation transition is structurally valid"),
);
let cancelled_notification = final_task_notification(&cancelled);
let cancelled_snapshot =
FinalTaskStore::request_cancellation_and_clear_input_if_current(
&*store,
&target_snapshot,
cancelled,
cancelled_notification,
)
.expect("live atomic cancellation CAS is readable")
.expect("live atomic cancellation commits");
assert!(matches!(cancelled_snapshot.task(), FinalTask::Cancelled(_)));
}
1 => {
assert!(
FinalTaskStore::request_cancellation_if_current(&*store, &target_snapshot)
.expect("live raw cancellation CAS is readable"),
"a live snapshot retains its raw cancellation capability"
);
}
2 => {
FinalTaskStore::request_cancellation(&*store, &target_id)
.expect("live direct cancellation records intent");
}
_ => unreachable!("all cancellation variants are covered"),
}
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(state.generations.get(&target_id), Some(&target_generation));
match variant {
0 => {
assert!(matches!(
state.tasks.get(&target_id),
Some(FinalTask::Cancelled(_))
));
assert!(!state.cancellation_requests.contains(&target_id));
assert!(state.latest_notifications.contains_key(&target_id));
}
1 | 2 => {
assert!(matches!(
state.tasks.get(&target_id),
Some(FinalTask::Working(_))
));
assert!(state.cancellation_requests.contains(&target_id));
assert!(state.latest_notifications.contains_key(&target_id));
}
_ => unreachable!("all cancellation variants are covered"),
}
assert_eq!(
serde_json::to_value(state.tasks.get(&sibling_id))
.expect("encode retained sibling task"),
serde_json::to_value(sibling_snapshot.task())
.expect("encode pre-expiry sibling task"),
"cancellation leaves the live sibling task unchanged"
);
assert_eq!(
state.generations.get(&sibling_id),
Some(&sibling_snapshot.generation())
);
assert_eq!(
serde_json::to_value(state.latest_notifications.get(&sibling_id))
.expect("encode retained sibling notification"),
serde_json::to_value(&sibling_notification)
.expect("encode pre-expiry sibling notification"),
"cancellation leaves the live sibling notification unchanged"
);
assert!(!state.cancellation_requests.contains(&sibling_id));
}
}
#[test]
fn task_03_final_public_predispatch_cancellation_commits_cancelled_task_and_notification() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let delivered = Arc::new(Mutex::new(Vec::new()));
let delivered_for_emitter = Arc::clone(&delivered);
let runtime_store: Arc<dyn FinalTaskStore> = store.clone();
let runtime = FinalTaskRuntime::new(
runtime_store,
FinalTaskRuntimeConfig::new(60_000, Some(5_000))
.expect("a finite final task policy is valid"),
Arc::new(move |notification| {
delivered_for_emitter
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(notification);
}),
);
let runner = runtime
.install_task_service(1, Arc::new(PendingFinalTaskSupervisor))
.expect("a task service is installed before public task creation");
let service_cx = Cx::for_testing();
let mut service = Box::pin(runner.run(&service_cx));
let mut task_cx = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
Future::poll(service.as_mut(), &mut task_cx),
std::task::Poll::Pending
));
let created = runtime
.create_task_with_work(final_test_work_descriptor(), Some("queued".to_owned()))
.expect("the entered service admits public initial work");
let task_id = created.task.base().task_id.clone();
let before = store
.get_task_snapshot(&task_id)
.expect("the created task snapshot is readable")
.expect("the created task is retained");
runtime
.cancel_task(&task_id)
.expect("pre-dispatch cancellation is acknowledged");
let after = store
.get_task_snapshot(&task_id)
.expect("the cancelled task snapshot is readable")
.expect("the cancelled task is retained");
assert!(matches!(after.task(), FinalTask::Cancelled(_)));
assert_eq!(
after.generation(),
before
.generation()
.checked_add(1)
.expect("the fixture generation remains representable"),
"the terminal cancellation is the task's sole post-create durable transition"
);
assert!(
!runtime
.is_cancellation_requested(&task_id)
.expect("terminal cancellation consumes cooperative intent")
);
assert!(
test_next_initial_work(&store)
.expect("cancelled initial-work recovery scan is valid")
.is_none(),
"terminal cancellation leaves no initial work for a supervisor"
);
let delivered = delivered
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(delivered.len(), 2, "creation and cancellation both notify");
assert!(delivered.last().is_some_and(|notification| {
matches!(¬ification.params.task, FinalTask::Cancelled(_))
}));
assert_eq!(
serde_json::to_value(
store
.latest_notification(&task_id)
.expect("the terminal cancellation notification is retained")
.params
.task,
)
.expect("encode retained cancellation notification"),
serde_json::to_value(after.task()).expect("encode retained cancelled task"),
"the durable terminal snapshot and emitted notification agree exactly"
);
}
#[test]
fn task_03_final_cancellation_clears_pending_handoff_before_supervisor_invocation() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses);
let delivered = Arc::new(Mutex::new(Vec::new()));
let mut runner = runtime
.install_task_service(
1,
Arc::new(RecordingFinalTaskSupervisor {
accepted: Arc::clone(&delivered),
}),
)
.expect("install caller-owned service for cancellation wakeup");
runtime
.cancel_task(&task_id)
.expect("cancellation atomically claims the pending handoff");
assert_eq!(
runner
.receiver
.try_recv()
.expect("cancellation wakes the installed task service"),
task_id
);
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
application_runtime
.block_on(runner.resume_task(&Cx::for_testing(), &task_id))
.expect("cancelled task wakeup does not invoke the supervisor");
assert!(
delivered
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty(),
"a cancellation that wins before the take cannot reach application code"
);
assert!(
runtime
.recover_accepted_input()
.expect("cancelled recovery scan is valid")
.is_none(),
"the cancellation transaction clears the durable accepted-input handoff"
);
}
#[test]
fn task_03_final_cancellation_fences_handoff_claimed_before_invocation() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses);
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted handoff before cancellation")
.expect("accepted handoff is present before cancellation");
let delivered = Arc::new(Mutex::new(Vec::new()));
let runner = runtime
.install_task_service(
1,
Arc::new(RecordingFinalTaskSupervisor {
accepted: Arc::clone(&delivered),
}),
)
.expect("install caller-owned service for dispatch fence");
runtime
.cancel_task(&task_id)
.expect("cancellation wins after handoff claim but before invocation");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
let cx = Cx::for_testing();
application_runtime
.block_on(runner.resume_handoff(&cx, FinalTaskSupervisorHandoff::Resumed(accepted)))
.expect("stale claimed handoff is fenced without invoking application work");
assert!(
delivered
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty(),
"the post-claim cancellation generation fence prevents cancelled work reaching the app"
);
}
#[test]
fn task_03_final_dispatch_election_delivers_uncancelled_handoff() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim uncancelled accepted handoff")
.expect("accepted handoff is present");
let delivered = Arc::new(Mutex::new(Vec::new()));
let runner = runtime
.install_task_service(
1,
Arc::new(RecordingFinalTaskSupervisor {
accepted: Arc::clone(&delivered),
}),
)
.expect("install caller-owned service for dispatch election");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
application_runtime
.block_on(runner.resume_handoff(
&Cx::for_testing(),
FinalTaskSupervisorHandoff::Resumed(accepted),
))
.expect("the uncancelled dispatch election reaches application work");
assert_eq!(
delivered
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_slice(),
&[(task_id, input_responses)],
"the elected handoff is delivered exactly once"
);
}
#[test]
fn task_03_final_cancellation_wins_atomic_dispatch_election() {
let inner = Arc::new(InMemoryFinalTaskStore::default());
let setup_runtime =
final_task_runtime(Arc::clone(&inner), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&setup_runtime, input_responses);
let accepted = setup_runtime
.take_accepted_input(&task_id)
.expect("claim handoff before the dispatch race")
.expect("accepted handoff is present before the dispatch race");
let runtime = FinalTaskRuntime::new(
Arc::new(CancelBeforeFinalTaskDispatchStore {
inner: Arc::clone(&inner),
}),
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid final task policy"),
Arc::new(|_| {}),
);
let delivered = Arc::new(Mutex::new(Vec::new()));
let runner = runtime
.install_task_service(
1,
Arc::new(RecordingFinalTaskSupervisor {
accepted: Arc::clone(&delivered),
}),
)
.expect("install caller-owned service for atomic dispatch race");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
application_runtime
.block_on(runner.resume_handoff(
&Cx::for_testing(),
FinalTaskSupervisorHandoff::Resumed(accepted),
))
.expect("a cancellation election loser returns without invoking application work");
assert!(
delivered
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty(),
"the cancellation that linearizes before dispatch cannot reach the supervisor"
);
assert!(
!runtime
.is_cancellation_requested(&task_id)
.expect("read terminal cancellation state"),
"the pre-dispatch terminal cancellation consumes cooperative intent"
);
}
#[test]
fn task_03_final_handoff_drop_restores_exact_resumed_input() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses = serde_json::from_value(
serde_json::json!({"roots": {"roots": [{"uri": "file:///drop-restore"}]}}),
)
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before cancellation-style future drop")
.expect("accepted input is present before the dropped supervisor future");
let runner = runtime
.install_task_service(1, Arc::new(PendingFinalTaskSupervisor))
.expect("install caller-owned pending service runner");
let cx = Cx::for_testing();
{
let pending = runner.resume_handoff(&cx, FinalTaskSupervisorHandoff::Resumed(accepted));
let mut pending = std::pin::pin!(pending);
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
std::future::Future::poll(pending.as_mut(), &mut context),
std::task::Poll::Pending
));
}
let restored = runtime
.recover_accepted_input()
.expect("dropped supervisor handoff recovery scan is valid")
.expect("drop lease restores the accepted input");
assert_eq!(restored.input_responses(), &input_responses);
}
#[test]
fn task_03_final_uncancelled_elected_handoff_error_requeues_under_unlimited_retention() {
const ELAPSED_MS: u64 = 86_400_000;
let (store, now) = execution_test_store(2 * ELAPSED_MS, 3_600_000, 8);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_unlimited_ttl(&AllowUnlimitedFinalTaskRetention, None)
.expect("explicit authority admits unlimited retained handoffs"),
Arc::new(|_| {}),
);
let input_responses: FinalTaskInputResponses = serde_json::from_value(
serde_json::json!({"roots": {"roots": [{"uri": "file:///uncancelled-error"}]}}),
)
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before the planted supervisor error")
.expect("accepted input is present before the planted supervisor error");
let runner = runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install caller-owned failing service runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
assert!(
application_runtime
.block_on(runner.resume_handoff(
&Cx::for_testing(),
FinalTaskSupervisorHandoff::Resumed(accepted),
))
.is_err(),
"the planted supervisor error remains visible after durable restoration"
);
assert!(
!runtime
.is_cancellation_requested(&task_id)
.expect("read uncancelled task state"),
"only cancellation differs from the paired fenced error path"
);
assert!(
!store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.handoff_leases
.contains_key(&task_id),
"an error restoration releases its elected owner fence before requeueing"
);
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(ELAPSED_MS))
.expect("test clock can advance through an unlimited retention interval");
drop(clock);
assert!(
runtime.get_task(&task_id).is_ok(),
"null-TTL retention keeps the task available after error recovery"
);
let restored = runtime
.recover_accepted_input()
.expect("recovery scan reads the uncancelled restored handoff")
.expect("uncancelled error requeues the exact accepted handoff");
assert_eq!(restored.input_responses(), &input_responses);
}
#[test]
fn task_03_final_cancelled_elected_handoff_error_retires_task_under_unlimited_retention() {
const ELAPSED_MS: u64 = 86_400_000;
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_unlimited_ttl(&AllowUnlimitedFinalTaskRetention, None)
.expect("explicit authority admits unlimited retained handoffs"),
Arc::new(|_| {}),
);
let input_responses: FinalTaskInputResponses = serde_json::from_value(
serde_json::json!({"roots": {"roots": [{"uri": "file:///cancelled-error"}]}}),
)
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses);
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before cancellation after dispatch election")
.expect("accepted input is present before cancellation after dispatch election");
let runner = runtime
.install_task_service(
1,
Arc::new(CancelThenFailingFinalTaskSupervisor {
runtime: runtime.clone(),
}),
)
.expect("install caller-owned cancelling failing service runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
application_runtime
.block_on(runner.resume_handoff(
&Cx::for_testing(),
FinalTaskSupervisorHandoff::Resumed(accepted),
))
.expect(
"the cancellation winner retires an elected task even when application work fails",
);
assert!(
!runtime
.is_cancellation_requested(&task_id)
.expect("read terminal cancellation state"),
"automatic retirement consumes the elected cancellation intent"
);
assert!(matches!(
runtime
.get_task(&task_id)
.expect("read automatically retired task")
.task,
FinalTask::Cancelled(_)
));
{
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
!state.handoff_leases.contains_key(&task_id),
"automatic cancellation retirement releases the exact elected owner fence"
);
assert!(
!state.accepted_inputs.contains_key(&task_id),
"cancellation, unlike the paired error path, must not requeue input"
);
}
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(ELAPSED_MS))
.expect("test clock can advance through an unlimited retention interval");
drop(clock);
assert!(
runtime.get_task(&task_id).is_ok(),
"unbounded task retention preserves the automatically retired terminal task"
);
assert!(
runtime
.recover_accepted_input()
.expect("cancelled recovery scan is readable")
.is_none(),
"cancelled input is never replayed after its elected owner releases the fence"
);
}
#[test]
fn task_03_final_cancelled_elected_handoff_drop_retires_task_under_unlimited_retention() {
const ELAPSED_MS: u64 = 86_400_000;
let (store, now) = in_memory_store_with_test_clock(1);
let runtime = FinalTaskRuntime::new(
store.clone(),
FinalTaskRuntimeConfig::with_unlimited_ttl(&AllowUnlimitedFinalTaskRetention, None)
.expect("explicit authority admits unlimited retained handoffs"),
Arc::new(|_| {}),
);
let input_responses: FinalTaskInputResponses = serde_json::from_value(
serde_json::json!({"roots": {"roots": [{"uri": "file:///cancelled-drop"}]}}),
)
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses);
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before dropped elected supervisor")
.expect("accepted input is present before dropped elected supervisor");
let runner = runtime
.install_task_service(1, Arc::new(PendingFinalTaskSupervisor))
.expect("install caller-owned pending service runner");
let cx = Cx::for_testing();
{
let pending = runner.resume_handoff(&cx, FinalTaskSupervisorHandoff::Resumed(accepted));
let mut pending = std::pin::pin!(pending);
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
std::future::Future::poll(pending.as_mut(), &mut context),
std::task::Poll::Pending
));
runtime
.cancel_task(&task_id)
.expect("an elected working task accepts cooperative cancellation");
}
assert!(
!runtime
.is_cancellation_requested(&task_id)
.expect("read terminal cancellation state after dropping the elected future"),
"the dropped elected handoff retires an already-recorded cancellation"
);
assert!(matches!(
runtime
.get_task(&task_id)
.expect("read automatically retired dropped task")
.task,
FinalTask::Cancelled(_)
));
{
let state = store
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
!state.handoff_leases.contains_key(&task_id),
"dropping a cancelled elected future retires and releases its exact owner fence"
);
assert!(
!state.accepted_inputs.contains_key(&task_id),
"cancellation prevents the dropped future from requeueing its input"
);
}
let mut clock = now
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*clock = clock
.checked_add(StdDuration::from_millis(ELAPSED_MS))
.expect("test clock can advance through an unlimited retention interval");
drop(clock);
assert!(
runtime.get_task(&task_id).is_ok(),
"unbounded retention keeps the retired cancelled task inspectable without its fence"
);
assert!(
runtime
.recover_accepted_input()
.expect("cancelled dropped-future recovery scan is readable")
.is_none(),
"a cancelled dropped future cannot replay retained application input"
);
}
#[test]
fn task_03_final_handoff_cancellation_checkpoint_restores_input_before_invocation() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before cancellation checkpoint")
.expect("accepted input is present before cancellation");
let runner = runtime
.install_task_service(1, Arc::new(PendingFinalTaskSupervisor))
.expect("install caller-owned pending service runner");
let cx = Cx::for_testing();
cx.cancel_with(CancelKind::User, None);
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
assert!(
application_runtime
.block_on(runner.resume_handoff(&cx, FinalTaskSupervisorHandoff::Resumed(accepted)))
.is_err(),
"the pre-invocation cancellation checkpoint stops application work"
);
let restored = runtime
.recover_accepted_input()
.expect("cancelled handoff recovery scan is valid")
.expect("cancellation drops the lease and restores the accepted input");
assert_eq!(restored.input_responses(), &input_responses);
}
#[test]
fn task_03_final_pending_supervisor_remains_owned_without_context_cancellation() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses);
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before pending supervisor start")
.expect("accepted input is present before the pending supervisor starts");
let runner = runtime
.install_task_service(1, Arc::new(PendingFinalTaskSupervisor))
.expect("install caller-owned pending service runner");
let cx = Cx::for_testing();
let pending = runner.resume_handoff(&cx, FinalTaskSupervisorHandoff::Resumed(accepted));
let mut pending = std::pin::pin!(pending);
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
std::future::Future::poll(pending.as_mut(), &mut context),
std::task::Poll::Pending
));
assert!(
runtime
.recover_accepted_input()
.expect("read pending supervisor recovery state")
.is_none(),
"without context cancellation the elected pending supervisor retains its durable lease"
);
}
#[test]
fn task_03_final_context_cancellation_after_pending_supervisor_start_restores_input() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before pending supervisor start")
.expect("accepted input is present before the pending supervisor starts");
let runner = runtime
.install_task_service(1, Arc::new(PendingFinalTaskSupervisor))
.expect("install caller-owned pending service runner");
let cx = Cx::for_testing();
let pending = runner.resume_handoff(&cx, FinalTaskSupervisorHandoff::Resumed(accepted));
let mut pending = std::pin::pin!(pending);
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
std::future::Future::poll(pending.as_mut(), &mut context),
std::task::Poll::Pending
));
cx.cancel_with(CancelKind::User, None);
assert!(matches!(
std::future::Future::poll(pending.as_mut(), &mut context),
std::task::Poll::Ready(Err(_))
));
let restored = runtime
.recover_accepted_input()
.expect("cancelled pending supervisor recovery scan is valid")
.expect("context cancellation restores the exact pending input handoff");
assert_eq!(restored.input_responses(), &input_responses);
}
#[test]
fn task_03_final_live_pending_supervisor_without_cancellation_retains_handoff() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before pending supervisor start")
.expect("accepted input is present before the pending supervisor starts");
let (started, mut started_receiver) = mpsc::channel(1);
let runner = runtime
.install_task_service(
1,
Arc::new(SignallingPendingFinalTaskSupervisor { started }),
)
.expect("install caller-owned pending service runner");
let runtime_for_task = runtime.clone();
let child_context = Arc::new(Mutex::new(None));
let child_context_for_task = Arc::clone(&child_context);
let ((), report) = asupersync::lab::run_async_under_lab(0x71_03, move |cx| async move {
let runner_context = Arc::clone(&child_context_for_task);
let mut supervisor = cx
.spawn(move |supervisor_cx| async move {
*runner_context
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(supervisor_cx.clone());
runner
.resume_handoff(
&supervisor_cx,
FinalTaskSupervisorHandoff::Resumed(accepted),
)
.await
})
.expect("live runtime admits the pending supervisor");
started_receiver
.recv(&cx)
.await
.expect("pending supervisor reports its first live poll");
let supervisor_cx = child_context_for_task
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref()
.expect("the live supervisor publishes its context before polling")
.clone();
assert!(
!supervisor_cx.is_cancel_requested(),
"only the absence of Cx cancellation differs from the paired wake path"
);
assert!(
runtime_for_task
.recover_accepted_input()
.expect("read live pending-supervisor recovery state")
.is_none(),
"without Cx cancellation the pending supervisor retains its durable lease"
);
supervisor_cx.cancel_with(CancelKind::User, None);
assert!(matches!(
supervisor.join(&cx).await,
Ok(Err(_)) | Err(asupersync::runtime::JoinError::Cancelled(_))
));
});
assert_eq!(
report.now_nanos, 0,
"the cleanup cancellation wakes the live task before its heartbeat timer"
);
let restored = runtime
.recover_accepted_input()
.expect("cancelled pending supervisor recovery scan is valid")
.expect("cleanup cancellation restores the exact pending input handoff");
assert_eq!(restored.input_responses(), &input_responses);
}
#[test]
fn task_03_final_task_cancellation_wakes_and_retires_pending_supervisor() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses);
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before task-cancellation wakeup")
.expect("accepted input is present before the pending supervisor starts");
let (started, mut started_receiver) = mpsc::channel(1);
let runner = runtime
.install_task_service(
1,
Arc::new(SignallingPendingFinalTaskSupervisor { started }),
)
.expect("install caller-owned pending service runner");
let runtime_for_task = runtime.clone();
let task_id_for_task = task_id.clone();
let ((), report) = asupersync::lab::run_async_under_lab(0x71_05, move |cx| async move {
let mut supervisor = cx
.spawn(move |supervisor_cx| async move {
runner
.resume_handoff(
&supervisor_cx,
FinalTaskSupervisorHandoff::Resumed(accepted),
)
.await
})
.expect("live runtime admits the pending supervisor");
started_receiver
.recv(&cx)
.await
.expect("pending supervisor reports its first live poll");
runtime_for_task
.cancel_task(&task_id_for_task)
.expect("tasks/cancel commits cancellation and wakes the elected handoff");
assert!(matches!(supervisor.join(&cx).await, Ok(Ok(()))));
});
assert_eq!(
report.now_nanos, 0,
"the durable task cancellation wakes the parked supervisor without waiting for a heartbeat"
);
assert!(matches!(
runtime
.get_task(&task_id)
.expect("read task after cancellation wakeup")
.task,
FinalTask::Cancelled(_)
));
assert!(
!runtime
.is_cancellation_requested(&task_id)
.expect("read terminal cancellation state"),
"automatic retirement consumes cancellation intent"
);
assert!(
runtime
.recover_accepted_input()
.expect("cancelled recovery scan is readable")
.is_none(),
"changing only the cancellation winner prevents the paired retained input from replaying"
);
}
#[test]
fn task_03_final_live_context_cancellation_wakes_pending_supervisor() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before pending supervisor start")
.expect("accepted input is present before the pending supervisor starts");
let (started, mut started_receiver) = mpsc::channel(1);
let runner = runtime
.install_task_service(
1,
Arc::new(SignallingPendingFinalTaskSupervisor { started }),
)
.expect("install caller-owned pending service runner");
let child_context = Arc::new(Mutex::new(None));
let child_context_for_task = Arc::clone(&child_context);
let ((), report) = asupersync::lab::run_async_under_lab(0x71_04, move |cx| async move {
let runner_context = Arc::clone(&child_context_for_task);
let mut supervisor = cx
.spawn(move |supervisor_cx| async move {
*runner_context
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(supervisor_cx.clone());
runner
.resume_handoff(
&supervisor_cx,
FinalTaskSupervisorHandoff::Resumed(accepted),
)
.await
})
.expect("live runtime admits the pending supervisor");
started_receiver
.recv(&cx)
.await
.expect("pending supervisor reports its first live poll");
child_context_for_task
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref()
.expect("the live supervisor publishes its context before polling")
.cancel_with(CancelKind::User, None);
assert!(matches!(
supervisor.join(&cx).await,
Ok(Err(_)) | Err(asupersync::runtime::JoinError::Cancelled(_))
));
});
assert_eq!(
report.now_nanos, 0,
"Cx cancellation wakes the live pending supervisor without waiting for a heartbeat"
);
let restored = runtime
.recover_accepted_input()
.expect("cancelled pending supervisor recovery scan is valid")
.expect("context cancellation restores the exact pending input handoff");
assert_eq!(restored.input_responses(), &input_responses);
}
#[test]
fn task_03_final_handoff_panic_restores_exact_resumed_input() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&runtime, input_responses.clone());
let accepted = runtime
.take_accepted_input(&task_id)
.expect("claim accepted input before planted panic")
.expect("accepted input is present before the panicking supervisor future");
let runner = runtime
.install_task_service(1, Arc::new(PanickingFinalTaskSupervisor))
.expect("install caller-owned panicking service runner");
let cx = Cx::for_testing();
let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let panicking =
runner.resume_handoff(&cx, FinalTaskSupervisorHandoff::Resumed(accepted));
let mut panicking = std::pin::pin!(panicking);
let mut context = std::task::Context::from_waker(std::task::Waker::noop());
let _ = std::future::Future::poll(panicking.as_mut(), &mut context);
}));
assert!(
panic.is_err(),
"the planted supervisor panic reaches the caller"
);
let restored = runtime
.recover_accepted_input()
.expect("panicking supervisor recovery scan is valid")
.expect("unwinding drops the lease and restores the accepted input");
assert_eq!(restored.input_responses(), &input_responses);
}
#[test]
fn task_03_final_service_runner_recovers_and_delivers_accepted_input() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let first_runtime =
final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses = serde_json::from_value(
serde_json::json!({"roots": {"roots": [{"uri": "file:///recovery-success"}]}}),
)
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&first_runtime, input_responses.clone());
drop(first_runtime);
let recovered_runtime =
final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let delivered = Arc::new(Mutex::new(Vec::new()));
let runner = recovered_runtime
.install_task_service(
1,
Arc::new(RecordingFinalTaskSupervisor {
accepted: Arc::clone(&delivered),
}),
)
.expect("install caller-owned service runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
let cx = Cx::for_testing();
application_runtime
.block_on(runner.run(&cx))
.expect("recovered accepted input reaches the supervisor");
let delivered = delivered
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
delivered.as_slice(),
&[(task_id.clone(), input_responses)],
"the recovery scan delivers the exact durable handoff once"
);
drop(delivered);
assert!(
recovered_runtime
.recover_accepted_input()
.expect("empty recovery scan is valid")
.is_none(),
"a successful supervisor call consumes the durable handoff"
);
}
#[test]
fn task_03_final_service_runner_continues_after_sixty_four_recoveries() {
let store = Arc::new(InMemoryFinalTaskStore::default());
for index in 0..=MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN {
let task = final_working_task_without_ttl(&format!("task-bounded-recovery-{index:03}"));
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.expect("every bounded-recovery fixture retains its initial work");
}
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let started = Arc::new(AtomicUsize::new(0));
let runner = runtime
.install_task_service(
1,
Arc::new(CancellingAfterInitialHandoffsFinalTaskSupervisor {
started: Arc::clone(&started),
cancel_after: MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN + 1,
}),
)
.expect("install caller-owned bounded recovery service runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
let cx = Cx::for_testing();
application_runtime
.block_on(runner.run(&cx))
.expect("a self-wakeup continues the bounded recovery scan");
assert_eq!(
started.load(AtomicOrdering::SeqCst),
MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN + 1,
"the sixty-fifth retained initial handoff runs in the continuation turn"
);
}
#[test]
fn task_03_final_recovery_interleaves_resumed_input_with_initial_backlog() {
let store = Arc::new(InMemoryFinalTaskStore::default());
for index in 0..=MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN {
let task = final_working_task_without_ttl(&format!("task-fair-initial-{index:03}"));
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.expect("every initial-backlog fixture retains durable work");
}
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
create_accepted_final_input(&runtime, input_responses);
let order = Arc::new(Mutex::new(Vec::new()));
let runner = runtime
.install_task_service(
1,
Arc::new(RecordingRecoveryOrderFinalTaskSupervisor {
order: Arc::clone(&order),
cancel_after: 2,
}),
)
.expect("install caller-owned fair recovery service runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
application_runtime
.block_on(runner.run(&Cx::for_testing()))
.expect("cancellation after the paired handoffs exits cleanly");
assert_eq!(
order
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_slice(),
["initial", "resumed"],
"a resumed input is delivered on the second bounded recovery claim despite the initial backlog"
);
}
#[test]
fn task_03_final_recovery_initial_only_backlog_never_fabricates_resumption() {
let store = Arc::new(InMemoryFinalTaskStore::default());
for index in 0..=MAX_FINAL_TASK_RECOVERY_HANDOFFS_PER_SCAN {
let task =
final_working_task_without_ttl(&format!("task-fair-initial-only-{index:03}"));
store
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.expect("every initial-only fixture retains durable work");
}
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let order = Arc::new(Mutex::new(Vec::new()));
let runner = runtime
.install_task_service(
1,
Arc::new(RecordingRecoveryOrderFinalTaskSupervisor {
order: Arc::clone(&order),
cancel_after: 2,
}),
)
.expect("install caller-owned initial-only recovery service runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
application_runtime
.block_on(runner.run(&Cx::for_testing()))
.expect("cancellation after two initial handoffs exits cleanly");
assert_eq!(
order
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_slice(),
["initial", "initial"],
"changing only the absence of accepted input preserves initial recovery without inventing a resumed handoff"
);
}
#[test]
fn task_03_final_service_runner_error_restores_exact_accepted_input() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let first_runtime =
final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses = serde_json::from_value(
serde_json::json!({"roots": {"roots": [{"uri": "file:///recovery-error"}]}}),
)
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&first_runtime, input_responses.clone());
drop(first_runtime);
let recovered_runtime =
final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let runner = recovered_runtime
.install_task_service(1, Arc::new(FailingFinalTaskSupervisor))
.expect("install caller-owned failing service runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
assert!(
application_runtime
.block_on(runner.run(&Cx::for_testing()))
.is_err(),
"the supervisor error remains visible after durable restoration"
);
let restored = recovered_runtime
.recover_accepted_input()
.expect("recovery scan reads restored handoff")
.expect("supervisor failure restores the accepted handoff");
assert_eq!(restored.task_id(), &task_id);
assert_eq!(
restored.input_responses(),
&input_responses,
"error recovery restores the exact input payload cloned before await"
);
}
#[test]
fn task_03_final_retryable_low_id_recovery_does_not_starve_later_work_across_runner_restart() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let low = final_working_task_without_ttl("task-recovery-a-retryable-low");
let high = final_working_task_without_ttl("task-recovery-b-later-work");
let low_id = low.base().task_id.clone();
let high_id = high.base().task_id.clone();
store
.create_task_with_work(
low.clone(),
final_task_notification(&low),
final_test_work_descriptor(),
)
.expect("the low-ID retry fixture is retained before the first runner starts");
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let first_attempted = Arc::new(Mutex::new(Vec::new()));
let first_runner = runtime
.install_task_service(
1,
Arc::new(FailLowIdCompleteLaterInitialSupervisor {
low_task_id: low_id.clone(),
attempted: Arc::clone(&first_attempted),
}),
)
.expect("first service runner installs");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
assert!(
application_runtime
.block_on(first_runner.run(&Cx::for_testing()))
.is_err(),
"the first low-ID-only runner exposes its retryable supervisor error"
);
assert_eq!(
first_attempted
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_slice(),
&[low_id.clone(), low_id.clone()],
"the one-task baseline retries only the restored low-ID handoff"
);
store
.create_task_with_work(
high.clone(),
final_task_notification(&high),
final_test_work_descriptor(),
)
.expect("later durable work is retained before the replacement runner starts");
let second_attempted = Arc::new(Mutex::new(Vec::new()));
let second_runner = runtime
.install_task_service(
1,
Arc::new(FailLowIdCompleteLaterInitialSupervisor {
low_task_id: low_id.clone(),
attempted: Arc::clone(&second_attempted),
}),
)
.expect("replacement service runner installs after the retryable exit");
assert!(
application_runtime
.block_on(second_runner.run(&Cx::for_testing()))
.is_err(),
"the replacement runner preserves the original retryable error after advancing past it"
);
assert_eq!(
second_attempted
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_slice(),
&[low_id.clone(), high_id.clone(), low_id.clone()],
"changing only the later task and restarting the runner reaches it before the low-ID retry repeats"
);
assert!(matches!(
runtime
.get_task(&high_id)
.expect("later task remains readable after the retryable failure")
.task,
FinalTask::Completed { .. }
));
assert!(matches!(
runtime
.get_task(&low_id)
.expect("retryable low-ID task remains readable")
.task,
FinalTask::Working(_)
));
}
#[test]
fn task_03_final_service_runner_newer_transition_wins_over_error_restore() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let first_runtime =
final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let input_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed retained roots response");
let task_id = create_accepted_final_input(&first_runtime, input_responses);
drop(first_runtime);
let recovered_runtime =
final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let runner = recovered_runtime
.install_task_service(
1,
Arc::new(TerminalTransitionThenFailingFinalTaskSupervisor {}),
)
.expect("install caller-owned transitioning failing service runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
assert!(
application_runtime
.block_on(runner.run(&Cx::for_testing()))
.is_err(),
"the supervisor error remains visible when a newer transition wins"
);
assert!(matches!(
recovered_runtime
.get_task(&task_id)
.expect("read terminal winner after failed supervisor")
.task,
FinalTask::Completed { .. }
));
assert!(
recovered_runtime
.recover_accepted_input()
.expect("recovery scan after terminal transition is valid")
.is_none(),
"the generation-fenced restore cannot resurrect input into the newer terminal state"
);
}
#[test]
fn task_03_final_public_service_rejects_success_without_transition_and_restores_work() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let work_descriptor = FinalTaskWorkDescriptor::new(serde_json::json!({
"operation": "must-remain-recoverable-after-noop-supervisor",
}))
.expect("a non-null task descriptor is valid");
let runner = runtime
.install_task_service(1, Arc::new(NoTransitionFinalTaskSupervisor))
.expect("the caller-owned service installs");
let service_cx = Cx::for_testing();
let mut service = Box::pin(runner.run(&service_cx));
let mut task_cx = std::task::Context::from_waker(std::task::Waker::noop());
assert!(matches!(
Future::poll(service.as_mut(), &mut task_cx),
std::task::Poll::Pending
));
let task_id = runtime
.create_task_with_work(work_descriptor.clone(), None)
.expect("the entered service admits public initial work")
.task
.base()
.task_id
.clone();
let error = match Future::poll(service.as_mut(), &mut task_cx) {
std::task::Poll::Ready(Err(error)) => error,
std::task::Poll::Ready(Ok(())) => {
panic!("a no-transition supervisor must not be accepted as successful")
}
std::task::Poll::Pending => {
panic!("the queued no-transition supervisor must resolve to a recovery error")
}
};
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
assert_eq!(
error.message,
"Final task supervisor returned success without a fenced task transition"
);
assert!(matches!(
store
.get_task(&task_id)
.expect("the unresolved task remains readable"),
Some(FinalTask::Working(_))
));
let recovered = runtime
.recover_initial_work()
.expect("the rejected handoff remains recoverable")
.expect("the exact initial work is restored for another service generation");
assert_eq!(recovered.task_id(), &task_id);
assert_eq!(recovered.work_descriptor(), &work_descriptor);
}
#[test]
fn task_03_final_elected_handoff_completion_commits_atomically() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let before = store
.get_task_snapshot(&task_id)
.expect("read task before elected completion")
.expect("fixture task is retained");
let initial = runtime
.recover_initial_work()
.expect("recover initial handoff")
.expect("fixture task retains initial work");
let runner = runtime
.install_task_service(1, Arc::new(FencedCompletingFinalTaskSupervisor))
.expect("install the elected handoff runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
application_runtime
.block_on(runner.resume_handoff(
&Cx::for_testing(),
FinalTaskSupervisorHandoff::Initial(initial),
))
.expect("the elected handoff completes the task");
let after = store
.get_task_snapshot(&task_id)
.expect("read task after elected completion")
.expect("completed task is retained");
assert!(matches!(after.task(), FinalTask::Completed { .. }));
assert!(
after.generation() > before.generation(),
"the fenced terminal transition advances the durable generation"
);
assert!(matches!(
store
.latest_notification(&task_id)
.expect("terminal notification is retained")
.params
.task,
FinalTask::Completed { .. }
));
}
#[test]
fn task_03_final_stale_handoff_fence_rejects_identical_completion_without_mutation() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let before = store
.get_task_snapshot(&task_id)
.expect("read task before stale-fence completion")
.expect("fixture task is retained");
let notification_before = serde_json::to_value(
store
.latest_notification(&task_id)
.expect("fixture task retains its working notification"),
)
.expect("encode notification before stale-fence completion");
let initial = runtime
.recover_initial_work()
.expect("recover initial handoff")
.expect("fixture task retains initial work");
let observed_error = Arc::new(Mutex::new(None));
let runner = runtime
.install_task_service(
1,
Arc::new(StaleFenceCompletingFinalTaskSupervisor {
store: Arc::clone(&store),
observed_error: Arc::clone(&observed_error),
}),
)
.expect("install the stale-fence handoff runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
let runner_error = application_runtime
.block_on(runner.resume_handoff(
&Cx::for_testing(),
FinalTaskSupervisorHandoff::Initial(initial),
))
.expect_err("an unfenced successful return must leave recovery visibly failed");
assert_eq!(
runner_error.code,
fastmcp_core::McpErrorCode::InternalError,
"the runner reports that the supervisor returned success without a valid transition"
);
let error = observed_error
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
.expect("the only changed dimension, dispatch fence, rejects completion");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
let after = store
.get_task_snapshot(&task_id)
.expect("read task after stale-fence completion")
.expect("fixture task remains retained");
assert_eq!(
after.generation(),
before.generation(),
"a stale fence cannot advance the durable task generation"
);
assert!(matches!(after.task(), FinalTask::Working(_)));
assert_eq!(
serde_json::to_value(
store
.latest_notification(&task_id)
.expect("stale completion preserves the notification"),
)
.expect("encode notification after stale-fence completion"),
notification_before,
"a stale fence cannot replace the durable task notification"
);
}
#[test]
fn task_03_final_stale_handoff_generation_rejects_completion_without_second_mutation() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let before = store
.get_task_snapshot(&task_id)
.expect("read task before the competing generation transition")
.expect("fixture task is retained");
let initial = runtime
.recover_initial_work()
.expect("recover initial handoff")
.expect("fixture task retains initial work");
let observed_error = Arc::new(Mutex::new(None));
let runner = runtime
.install_task_service(
1,
Arc::new(StaleGenerationCompletingFinalTaskSupervisor {
runtime: runtime.clone(),
observed_error: Arc::clone(&observed_error),
}),
)
.expect("install the stale-generation handoff runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
application_runtime
.block_on(runner.resume_handoff(
&Cx::for_testing(),
FinalTaskSupervisorHandoff::Initial(initial),
))
.expect("the supervisor records its stale-generation rejection");
let error = observed_error
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
.expect("the stale generation rejects the terminal handoff");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
let after = store
.get_task_snapshot(&task_id)
.expect("read task after stale-generation completion")
.expect("the competing task state remains retained");
assert_eq!(
after.generation(),
before
.generation()
.checked_add(1)
.expect("fixture generation remains representable"),
"the rejected stale handoff cannot add a second durable transition"
);
assert!(matches!(
after.task(),
FinalTask::InputRequired { base, input_requests }
if base.status_message.as_deref()
== Some("newer generation won before completion")
&& input_requests.contains_key("roots")
));
assert_eq!(
serde_json::to_value(
store
.latest_notification(&task_id)
.expect("the competing notification remains retained")
.params
.task,
)
.expect("encode notification after stale-generation rejection"),
serde_json::to_value(after.task())
.expect("encode competing task after stale-generation rejection"),
"the stale handoff cannot replace the newer task notification"
);
}
#[test]
fn task_03_final_repeated_terminal_handoff_is_rejected_without_second_mutation() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let before = store
.get_task_snapshot(&task_id)
.expect("read task before repeated terminal handoff")
.expect("fixture task is retained");
let initial = runtime
.recover_initial_work()
.expect("recover initial handoff")
.expect("fixture task retains initial work");
let observed_error = Arc::new(Mutex::new(None));
let runner = runtime
.install_task_service(
1,
Arc::new(RepeatedTerminalHandoffSupervisor {
observed_error: Arc::clone(&observed_error),
}),
)
.expect("install repeated-terminal handoff runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
application_runtime
.block_on(runner.resume_handoff(
&Cx::for_testing(),
FinalTaskSupervisorHandoff::Initial(initial),
))
.expect("the repeated terminal attempt is contained by the supervisor");
let error = observed_error
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
.expect("the second terminal handoff is rejected");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InvalidParams);
let after = store
.get_task_snapshot(&task_id)
.expect("read task after repeated terminal handoff")
.expect("the first terminal result remains retained");
assert_eq!(
after.generation(),
before
.generation()
.checked_add(1)
.expect("fixture generation remains representable"),
"the repeated terminal handoff cannot add a second durable transition"
);
assert!(matches!(
after.task(),
FinalTask::Completed { base, .. }
if base.status_message.as_deref() == Some("first terminal handoff")
));
assert_eq!(
serde_json::to_value(
store
.latest_notification(&task_id)
.expect("the first terminal notification remains retained")
.params
.task,
)
.expect("encode notification after repeated terminal rejection"),
serde_json::to_value(after.task())
.expect("encode first terminal task after repeated rejection"),
"the repeated handoff cannot replace the first terminal notification"
);
}
#[test]
fn task_03_final_cancellation_winner_can_be_honored_by_the_elected_handoff() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let initial = runtime
.recover_initial_work()
.expect("recover initial handoff")
.expect("fixture task retains initial work");
let observed_cancellation = Arc::new(AtomicBool::new(false));
let runner = runtime
.install_task_service(
1,
Arc::new(CancelThenHonoringCancellationFinalTaskSupervisor {
runtime: runtime.clone(),
observed_cancellation: Arc::clone(&observed_cancellation),
}),
)
.expect("install cancellation-honouring handoff runner");
let application_runtime = RuntimeBuilder::current_thread()
.build()
.expect("build application-owned structured runtime");
application_runtime
.block_on(runner.resume_handoff(
&Cx::for_testing(),
FinalTaskSupervisorHandoff::Initial(initial),
))
.expect("the elected handoff records the cancellation outcome");
assert!(
observed_cancellation.load(AtomicOrdering::SeqCst),
"the elected handoff observes the cancellation winner before its terminal transition"
);
let after = store
.get_task_snapshot(&task_id)
.expect("read task after honoured cancellation")
.expect("cancelled task remains retained");
assert!(matches!(
after.task(),
FinalTask::Cancelled(base)
if base.status_message.as_deref()
== Some("cancellation won the elected handoff")
));
assert!(
!runtime
.is_cancellation_requested(&task_id)
.expect("terminal cancellation outcome clears the pending intent"),
"the final cancellation result consumes the cooperative intent"
);
}
#[test]
fn task_03_final_external_store_default_fenced_transition_fails_unchanged() {
let inner = Arc::new(InMemoryFinalTaskStore::default());
let external_store = ReadinessLeaseProbeFinalTaskStore::new(Arc::clone(&inner));
let task = final_working_task_without_ttl("task-external-default-fence");
let task_id = task.base().task_id.clone();
inner
.create_task_with_work(
task.clone(),
final_task_notification(&task),
final_test_work_descriptor(),
)
.expect("external-store fixture retains the initial task and work");
let expected = inner
.get_task_snapshot(&task_id)
.expect("read external-store fixture snapshot")
.expect("external-store fixture task is retained");
test_take_initial_work(&inner, &expected)
.expect("claim external-store fixture initial handoff")
.expect("external-store fixture retains initial work");
let dispatch_fence = FinalTaskStore::begin_handoff_dispatch_for_owner_if_current(
&*inner,
&task_id,
expected.generation(),
FINAL_TASK_TEST_DIRECT_OWNER,
)
.expect("elect external-store fixture handoff")
.expect("external-store fixture handoff election succeeds");
let task_before = serde_json::to_value(expected.task())
.expect("encode external-store task before default rejection");
let notification_before = serde_json::to_value(
inner
.latest_notification(&task_id)
.expect("external-store fixture notification is retained"),
)
.expect("encode external-store notification before default rejection");
let result: FinalTaskCallToolResult =
serde_json::from_value(serde_json::json!({"content": []}))
.expect("typed terminal task result");
let replacement = FinalTask::Completed {
base: transition_terminal_final_task_base(
expected.task().base().clone(),
FinalTaskStatus::Completed,
Some("default fenced transition must fail".to_owned()),
)
.expect("construct terminal replacement for the external-store probe"),
result,
};
let error = FinalTaskStore::replace_task_and_clear_input_for_handoff_if_current(
&external_store,
&expected,
FINAL_TASK_TEST_DIRECT_OWNER,
dispatch_fence,
false,
replacement.clone(),
final_task_notification(&replacement),
)
.expect_err("an external store must opt into fenced handoff transitions");
assert_eq!(error.code, fastmcp_core::McpErrorCode::InternalError);
let after = inner
.get_task_snapshot(&task_id)
.expect("read external-store fixture after default rejection")
.expect("default rejection leaves the task retained");
assert_eq!(after.generation(), expected.generation());
assert_eq!(
serde_json::to_value(after.task())
.expect("encode external-store task after default rejection"),
task_before,
"the default fenced-transition rejection leaves the task unchanged"
);
assert_eq!(
serde_json::to_value(
inner
.latest_notification(&task_id)
.expect("default rejection leaves the notification retained"),
)
.expect("encode external-store notification after default rejection"),
notification_before,
"the default fenced-transition rejection leaves the notification unchanged"
);
let state = inner
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
state.handoff_leases.get(&task_id).is_some_and(|lease| {
lease.generation == expected.generation()
&& lease.owner_id == FINAL_TASK_TEST_DIRECT_OWNER
&& lease.dispatch_elected
&& lease.dispatch_fence == Some(dispatch_fence)
}),
"the default fenced-transition rejection leaves the elected handoff lease unchanged"
);
}
#[test]
fn task_03_final_rejected_input_preserves_supervisor_handoff_state() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
runtime
.require_input(&task_id, final_roots_request(), None)
.expect("task awaits one roots response");
let before = serde_json::to_vec(
&runtime
.get_task(&task_id)
.expect("read input-required task before planted response")
.task,
)
.expect("serialize input-required task before planted response");
let notification_before = store
.latest_notification(&task_id)
.expect("input-required task retains its notification before planted response");
let wrong_kind: FinalTaskInputResponses = serde_json::from_value(serde_json::json!({
"roots": {
"role": "assistant",
"model": "final-model",
"content": {"type": "text", "text": "wrong response kind"}
}
}))
.expect("well-formed mismatched typed response");
assert!(
runtime.update_task(&task_id, &wrong_kind).is_err(),
"a mismatched response kind fails before it can reach the supervisor"
);
let after = serde_json::to_vec(
&runtime
.get_task(&task_id)
.expect("read task after rejected response")
.task,
)
.expect("serialize task after rejected response");
assert_eq!(
after, before,
"rejected input leaves durable task state unchanged"
);
assert_eq!(
serde_json::to_vec(&store.latest_notification(&task_id))
.expect("serialize retained notification after rejection"),
serde_json::to_vec(&Some(notification_before))
.expect("serialize baseline notification"),
"rejected input cannot replace the retained notification"
);
assert!(
runtime
.take_accepted_input(&task_id)
.expect("read unchanged supervisor handoff state")
.is_none(),
"rejected input cannot create a supervisor handoff"
);
}
#[test]
fn task_03_final_update_ignores_unknown_and_already_satisfied_keys_without_mutation() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let mut requests = final_roots_request();
requests.insert(
"other-roots".to_owned(),
serde_json::from_value(serde_json::json!({"method": "roots/list"}))
.expect("typed second roots request"),
);
runtime
.require_input(&task_id, requests, None)
.expect("task awaits two typed roots responses");
let first_response: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed first roots response");
runtime
.update_task(&task_id, &first_response)
.expect("first outstanding input response is accepted");
let before = serde_json::to_value(
&runtime
.get_task(&task_id)
.expect("read task before ignored replay")
.task,
)
.expect("encode task before ignored replay");
let generation_before = store
.get_task_snapshot(&task_id)
.expect("read task generation before ignored replay")
.expect("task is retained before ignored replay")
.generation();
let notification_before = serde_json::to_value(store.latest_notification(&task_id))
.expect("encode notification before ignored replay");
let ignored_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({
"roots": {
"role": "assistant",
"model": "final-model",
"content": {"type": "text", "text": "already satisfied"}
},
"stale-key": {
"role": "assistant",
"model": "final-model",
"content": {"type": "text", "text": "unknown"}
}
}))
.expect("well-formed ignored response map");
runtime
.update_task(&task_id, &ignored_responses)
.expect("unknown and already-satisfied input keys are acknowledged as a no-op");
assert_eq!(
serde_json::to_value(
&runtime
.get_task(&task_id)
.expect("read task after ignored replay")
.task,
)
.expect("encode task after ignored replay"),
before,
"ignored keys cannot change outstanding input state"
);
assert_eq!(
store
.get_task_snapshot(&task_id)
.expect("read generation after ignored replay")
.expect("task remains retained after ignored replay")
.generation(),
generation_before,
"ignored keys cannot advance the durable task generation"
);
assert_eq!(
serde_json::to_value(store.latest_notification(&task_id))
.expect("encode notification after ignored replay"),
notification_before,
"ignored keys cannot emit a replacement task notification"
);
}
#[test]
fn task_03_final_update_rejects_wrong_kind_for_outstanding_key_with_ignored_keys_present() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
let mut requests = final_roots_request();
requests.insert(
"other-roots".to_owned(),
serde_json::from_value(serde_json::json!({"method": "roots/list"}))
.expect("typed second roots request"),
);
runtime
.require_input(&task_id, requests, None)
.expect("task awaits two typed roots responses");
let first_response: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({"roots": {"roots": []}}))
.expect("typed first roots response");
runtime
.update_task(&task_id, &first_response)
.expect("first outstanding input response is accepted");
let before = serde_json::to_value(
&runtime
.get_task(&task_id)
.expect("read task before planted wrong-kind response")
.task,
)
.expect("encode task before planted wrong-kind response");
let generation_before = store
.get_task_snapshot(&task_id)
.expect("read generation before planted wrong-kind response")
.expect("task is retained before planted wrong-kind response")
.generation();
let rejected_responses: FinalTaskInputResponses =
serde_json::from_value(serde_json::json!({
"roots": {
"role": "assistant",
"model": "final-model",
"content": {"type": "text", "text": "already satisfied"}
},
"other-roots": {
"role": "assistant",
"model": "final-model",
"content": {"type": "text", "text": "wrong outstanding kind"}
}
}))
.expect("well-formed mixed response map");
assert!(
runtime.update_task(&task_id, &rejected_responses).is_err(),
"changing only the stale key to an outstanding roots key preserves wrong-kind rejection"
);
assert_eq!(
serde_json::to_value(
&runtime
.get_task(&task_id)
.expect("read task after rejected response")
.task,
)
.expect("encode task after rejected response"),
before,
"wrong-kind outstanding input cannot mutate task state"
);
assert_eq!(
store
.get_task_snapshot(&task_id)
.expect("read generation after rejected response")
.expect("task remains retained after rejected response")
.generation(),
generation_before,
"wrong-kind outstanding input cannot advance the task generation"
);
}
#[test]
fn task_03_final_durable_runtime_wrong_response_kind_preserves_state() {
let store = Arc::new(InMemoryFinalTaskStore::default());
let runtime = final_task_runtime(Arc::clone(&store), Arc::new(AtomicBool::new(false)));
let task_id = create_final_task_state_fixture(&runtime, None)
.task
.base()
.task_id
.clone();
runtime
.require_input(&task_id, final_roots_request(), None)
.expect("task awaits a roots response");
let before = serde_json::to_vec(
&runtime
.get_task(&task_id)
.expect("snapshot task before planted response")
.task,
)
.expect("serialize task snapshot");
let notification_before = store
.latest_notification(&task_id)
.expect("input-required task retains a typed notification");
let wrong_kind: FinalTaskInputResponses = serde_json::from_value(serde_json::json!({
"roots": {
"role": "assistant",
"model": "final-model",
"content": {"type": "text", "text": "wrong response kind"}
}
}))
.expect("well-formed but mismatched typed response");
assert!(
runtime.update_task(&task_id, &wrong_kind).is_err(),
"a response whose type does not match the issued request fails closed"
);
let after = serde_json::to_vec(
&runtime
.get_task(&task_id)
.expect("snapshot task after rejected response")
.task,
)
.expect("serialize task snapshot");
assert_eq!(
after, before,
"rejected input cannot mutate durable task state"
);
assert_eq!(
serde_json::to_vec(&store.latest_notification(&task_id))
.expect("serialize retained notification"),
serde_json::to_vec(&Some(notification_before))
.expect("serialize baseline notification"),
"rejected input cannot replace the retained typed notification"
);
}
#[test]
fn test_task_manager_creation() {
let manager = TaskManager::new();
assert_eq!(manager.total_count(), 0);
assert_eq!(manager.active_count(), 0);
assert!(!manager.has_list_changed_notifications());
}
#[test]
fn test_task_manager_with_notifications() {
let manager = TaskManager::with_list_changed_notifications();
assert!(manager.has_list_changed_notifications());
}
#[test]
fn test_register_handler() {
let manager = TaskManager::new();
manager.register_handler("test_task", |_cx, _params| async {
Ok(serde_json::json!({}))
});
let cx = Cx::for_testing();
let result = manager.submit(&cx, "test_task", None);
assert!(result.is_ok());
}
#[test]
fn test_submit_auto_execute_fails_when_runtime_unavailable() {
let mut manager = TaskManager::new_for_testing();
manager.auto_execute = true;
manager.runtime = None;
manager.register_handler("test_task", |_cx, _params| async {
Ok(serde_json::json!({}))
});
let cx = Cx::for_testing();
let task_id = manager.submit(&cx, "test_task", None).unwrap();
let info = manager.get_info(&task_id).unwrap();
assert_eq!(info.status, TaskStatus::Failed);
assert_eq!(info.error.as_deref(), Some("Task runtime unavailable"));
let result = manager.get_result(&task_id).unwrap();
assert!(!result.success);
assert_eq!(result.error.as_deref(), Some("Task runtime unavailable"));
}
#[test]
fn test_submit_unknown_task_type() {
let manager = TaskManager::new();
let cx = Cx::for_testing();
let result = manager.submit(&cx, "unknown_task", None);
assert!(result.is_err());
}
#[test]
fn test_task_lifecycle() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("test", |_cx, _params| async {
Ok(serde_json::json!({"done": true}))
});
let task_id = manager.submit(&cx, "test", None).unwrap();
let info = manager.get_info(&task_id).unwrap();
assert_eq!(info.status, TaskStatus::Pending);
assert!(info.started_at.is_none());
manager.start_task(&task_id).unwrap();
let info = manager.get_info(&task_id).unwrap();
assert_eq!(info.status, TaskStatus::Running);
assert!(info.started_at.is_some());
manager.update_progress(&task_id, 0.5, Some("Halfway done".into()));
let info = manager.get_info(&task_id).unwrap();
assert_eq!(info.progress, Some(0.5));
assert_eq!(info.message, Some("Halfway done".into()));
manager.complete_task(&task_id, serde_json::json!({"result": 42}));
let info = manager.get_info(&task_id).unwrap();
assert_eq!(info.status, TaskStatus::Completed);
assert!(info.completed_at.is_some());
let result = manager.get_result(&task_id).unwrap();
assert!(result.success);
assert_eq!(result.data, Some(serde_json::json!({"result": 42})));
}
#[test]
fn test_task_failure() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("fail_test", |_cx, _params| async {
Ok(serde_json::json!({}))
});
let task_id = manager.submit(&cx, "fail_test", None).unwrap();
manager.start_task(&task_id).unwrap();
manager.fail_task(&task_id, "Something went wrong");
let info = manager.get_info(&task_id).unwrap();
assert_eq!(info.status, TaskStatus::Failed);
assert_eq!(info.error, Some("Something went wrong".into()));
let result = manager.get_result(&task_id).unwrap();
assert!(!result.success);
assert_eq!(result.error, Some("Something went wrong".into()));
}
#[test]
fn test_task_cancellation() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("cancel_test", |_cx, _params| async {
Ok(serde_json::json!({}))
});
let task_id = manager.submit(&cx, "cancel_test", None).unwrap();
manager.start_task(&task_id).unwrap();
let info = manager
.cancel(&task_id, Some("User cancelled".into()))
.unwrap();
assert_eq!(info.status, TaskStatus::Cancelled);
assert!(manager.is_cancel_requested(&task_id));
let result = manager.cancel(&task_id, None);
assert!(result.is_err());
}
#[test]
fn test_list_tasks() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("list_test", |_cx, _params| async {
Ok(serde_json::json!({}))
});
let task1 = manager.submit(&cx, "list_test", None).unwrap();
let task2 = manager.submit(&cx, "list_test", None).unwrap();
let _task3 = manager.submit(&cx, "list_test", None).unwrap();
assert_eq!(manager.list_tasks(Some(TaskStatus::Pending)).len(), 3);
assert_eq!(manager.list_tasks(Some(TaskStatus::Running)).len(), 0);
manager.start_task(&task1).unwrap();
assert_eq!(manager.list_tasks(Some(TaskStatus::Pending)).len(), 2);
assert_eq!(manager.list_tasks(Some(TaskStatus::Running)).len(), 1);
manager.start_task(&task2).unwrap();
manager.complete_task(&task2, serde_json::json!({}));
assert_eq!(manager.list_tasks(Some(TaskStatus::Completed)).len(), 1);
assert_eq!(manager.list_tasks(None).len(), 3);
}
#[test]
fn test_active_count() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("count_test", |_cx, _params| async {
Ok(serde_json::json!({}))
});
let task1 = manager.submit(&cx, "count_test", None).unwrap();
let task2 = manager.submit(&cx, "count_test", None).unwrap();
assert_eq!(manager.active_count(), 2);
assert_eq!(manager.total_count(), 2);
manager.start_task(&task1).unwrap();
assert_eq!(manager.active_count(), 2);
manager.complete_task(&task1, serde_json::json!({}));
assert_eq!(manager.active_count(), 1);
manager.cancel(&task2, None).unwrap();
assert_eq!(manager.active_count(), 0);
assert_eq!(manager.total_count(), 2);
}
#[test]
fn test_progress_clamping() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("clamp_test", |_cx, _params| async {
Ok(serde_json::json!({}))
});
let task_id = manager.submit(&cx, "clamp_test", None).unwrap();
manager.start_task(&task_id).unwrap();
manager.update_progress(&task_id, -0.5, None);
assert_eq!(manager.get_info(&task_id).unwrap().progress, Some(0.0));
manager.update_progress(&task_id, 1.5, None);
assert_eq!(manager.get_info(&task_id).unwrap().progress, Some(1.0));
manager.update_progress(&task_id, 0.75, None);
assert_eq!(manager.get_info(&task_id).unwrap().progress, Some(0.75));
}
#[test]
fn test_invalid_transition_rejected() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("transition_test", |_cx, _params| async {
Ok(serde_json::json!({}))
});
let task_id = manager.submit(&cx, "transition_test", None).unwrap();
manager.complete_task(&task_id, serde_json::json!({"result": "noop"}));
let info = manager.get_info(&task_id).unwrap();
assert_eq!(info.status, TaskStatus::Pending);
manager.start_task(&task_id).unwrap();
manager.complete_task(&task_id, serde_json::json!({"result": "ok"}));
let info = manager.get_info(&task_id).unwrap();
assert_eq!(info.status, TaskStatus::Completed);
let result = manager.start_task(&task_id);
assert!(result.is_err());
}
#[test]
fn test_concurrent_submissions() {
let manager = Arc::new(TaskManager::new_for_testing());
manager.register_handler("concurrent_test", |_cx, _params| async {
Ok(serde_json::json!({}))
});
let mut handles = Vec::new();
for _ in 0..4 {
let manager = Arc::clone(&manager);
handles.push(thread::spawn(move || {
let cx = Cx::for_testing();
for _ in 0..10 {
let _ = manager.submit(&cx, "concurrent_test", None).unwrap();
}
}));
}
for handle in handles {
handle.join().expect("thread join failed");
}
assert_eq!(manager.total_count(), 40);
assert_eq!(manager.list_tasks(Some(TaskStatus::Pending)).len(), 40);
}
#[test]
fn test_task_status_notifications() {
let manager = TaskManager::new_for_testing();
manager.register_handler("notify_test", |_cx, _params| async {
Ok(serde_json::json!({"ok": true}))
});
let events: Arc<std::sync::Mutex<Vec<TaskStatusNotificationParams>>> =
Arc::new(std::sync::Mutex::new(Vec::new()));
let sender_events = Arc::clone(&events);
let sender: TaskNotificationSender = Arc::new(move |request| {
if request.method != "notifications/tasks/status" {
return;
}
let params = request
.params
.as_ref()
.and_then(|value| serde_json::from_value(value.clone()).ok())
.expect("task status params");
sender_events
.lock()
.expect("events lock poisoned")
.push(params);
});
manager.set_notification_sender(sender);
let cx = Cx::for_testing();
let task_id = manager.submit(&cx, "notify_test", None).unwrap();
manager.start_task(&task_id).unwrap();
manager.update_progress(&task_id, 0.5, Some("half".to_string()));
manager.complete_task(&task_id, serde_json::json!({"result": 1}));
let recorded = events.lock().expect("events lock poisoned").clone();
assert!(!recorded.is_empty(), "expected task status notifications");
assert_eq!(recorded[0].id, task_id);
assert_eq!(recorded[0].status, TaskStatus::Pending);
assert_eq!(recorded[1].status, TaskStatus::Running);
assert_eq!(recorded[2].progress, Some(0.5));
assert_eq!(recorded.last().expect("last").status, TaskStatus::Completed);
}
#[test]
fn can_transition_valid_pairs() {
assert!(can_transition(TaskStatus::Pending, TaskStatus::Running));
assert!(can_transition(TaskStatus::Pending, TaskStatus::Failed));
assert!(can_transition(TaskStatus::Pending, TaskStatus::Cancelled));
assert!(can_transition(TaskStatus::Running, TaskStatus::Completed));
assert!(can_transition(TaskStatus::Running, TaskStatus::Failed));
assert!(can_transition(TaskStatus::Running, TaskStatus::Cancelled));
}
#[test]
fn can_transition_invalid_pairs() {
assert!(!can_transition(TaskStatus::Pending, TaskStatus::Completed));
assert!(!can_transition(TaskStatus::Completed, TaskStatus::Running));
assert!(!can_transition(TaskStatus::Completed, TaskStatus::Pending));
assert!(!can_transition(
TaskStatus::Completed,
TaskStatus::Cancelled
));
assert!(!can_transition(TaskStatus::Failed, TaskStatus::Running));
assert!(!can_transition(TaskStatus::Cancelled, TaskStatus::Running));
}
#[test]
fn default_creates_empty_manager() {
let manager = TaskManager::default();
assert_eq!(manager.total_count(), 0);
assert!(!manager.has_list_changed_notifications());
}
#[test]
fn new_for_testing_disables_auto_execute() {
let manager = TaskManager::new_for_testing();
assert!(!manager.auto_execute);
}
#[test]
fn into_shared_returns_arc() {
let manager = TaskManager::new_for_testing();
let shared: SharedTaskManager = manager.into_shared();
assert_eq!(shared.total_count(), 0);
}
#[test]
fn debug_output_contains_fields() {
let manager = TaskManager::new_for_testing();
let debug = format!("{:?}", manager);
assert!(debug.contains("TaskManager"));
assert!(debug.contains("task_count"));
assert!(debug.contains("handler_count"));
assert!(debug.contains("task_counter"));
assert!(debug.contains("list_changed_notifications"));
assert!(debug.contains("auto_execute"));
}
#[test]
fn get_info_nonexistent_returns_none() {
let manager = TaskManager::new_for_testing();
let fake_id = TaskId::from_string("nonexistent".to_string());
assert!(manager.get_info(&fake_id).is_none());
}
#[test]
fn get_result_nonexistent_returns_none() {
let manager = TaskManager::new_for_testing();
let fake_id = TaskId::from_string("nonexistent".to_string());
assert!(manager.get_result(&fake_id).is_none());
}
#[test]
fn get_result_pending_task_returns_none() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
assert!(manager.get_result(&id).is_none());
}
#[test]
fn is_cancel_requested_nonexistent_returns_false() {
let manager = TaskManager::new_for_testing();
let fake_id = TaskId::from_string("nonexistent".to_string());
assert!(!manager.is_cancel_requested(&fake_id));
}
#[test]
fn is_cancel_requested_before_cancel_returns_false() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
assert!(!manager.is_cancel_requested(&id));
}
#[test]
fn update_progress_on_pending_task_is_ignored() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.update_progress(&id, 0.5, Some("test".to_string()));
let info = manager.get_info(&id).unwrap();
assert!(info.progress.is_none());
}
#[test]
fn update_progress_on_completed_task_is_ignored() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.complete_task(&id, serde_json::json!({}));
manager.update_progress(&id, 0.1, None);
let info = manager.get_info(&id).unwrap();
assert_eq!(info.progress, Some(1.0)); }
#[test]
fn complete_task_nonexistent_does_not_panic() {
let manager = TaskManager::new_for_testing();
let fake_id = TaskId::from_string("nonexistent".to_string());
manager.complete_task(&fake_id, serde_json::json!({})); }
#[test]
fn fail_task_nonexistent_does_not_panic() {
let manager = TaskManager::new_for_testing();
let fake_id = TaskId::from_string("nonexistent".to_string());
manager.fail_task(&fake_id, "error"); }
#[test]
fn cancel_nonexistent_task_returns_error() {
let manager = TaskManager::new_for_testing();
let fake_id = TaskId::from_string("nonexistent".to_string());
let err = manager.cancel(&fake_id, None).unwrap_err();
assert!(err.message.contains("not found"));
}
#[test]
fn cancel_pending_task_directly() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
let info = manager.cancel(&id, None).unwrap();
assert_eq!(info.status, TaskStatus::Cancelled);
assert!(manager.is_cancel_requested(&id));
}
#[test]
fn cancel_with_default_reason() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
let info = manager.cancel(&id, None).unwrap();
assert_eq!(info.error, Some("Cancelled by request".to_string()));
}
#[test]
fn task_ids_are_sequential() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id1 = manager.submit(&cx, "t", None).unwrap();
let id2 = manager.submit(&cx, "t", None).unwrap();
assert_ne!(id1, id2);
assert!(id1.0.starts_with("task-"));
assert!(id2.0.starts_with("task-"));
}
#[test]
fn start_task_nonexistent_returns_error() {
let manager = TaskManager::new_for_testing();
let fake_id = TaskId::from_string("nonexistent".to_string());
let err = manager.start_task(&fake_id).unwrap_err();
assert!(err.message.contains("not found"));
}
#[test]
fn start_task_already_running_returns_error() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
let err = manager.start_task(&id).unwrap_err();
assert!(err.message.contains("not pending"));
}
#[test]
fn cleanup_completed_removes_old_terminal_tasks() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.complete_task(&id, serde_json::json!({}));
assert_eq!(manager.total_count(), 1);
manager.cleanup_completed(std::time::Duration::from_secs(0));
assert_eq!(manager.total_count(), 0);
}
#[test]
fn cleanup_completed_keeps_active_tasks() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id1 = manager.submit(&cx, "t", None).unwrap();
let id2 = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id1).unwrap();
manager.complete_task(&id1, serde_json::json!({}));
manager.cleanup_completed(std::time::Duration::from_secs(0));
assert_eq!(manager.total_count(), 1); assert!(manager.get_info(&id2).is_some());
}
#[test]
fn cleanup_completed_keeps_recent_tasks() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.complete_task(&id, serde_json::json!({}));
manager.cleanup_completed(std::time::Duration::from_secs(3600));
assert_eq!(manager.total_count(), 1);
}
#[test]
fn transition_same_state_returns_true() {
let task_id = TaskId::from_string("test".to_string());
let mut state = TaskState {
info: TaskInfo {
id: task_id,
task_type: "t".to_string(),
status: TaskStatus::Running,
progress: None,
message: None,
created_at: String::new(),
started_at: None,
completed_at: None,
error: None,
},
cancel_requested: false,
result: None,
cx: Cx::for_testing(),
};
assert!(transition_state(&mut state, TaskStatus::Running));
}
#[test]
fn submit_with_none_params_creates_task() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
let info = manager.get_info(&id).unwrap();
assert_eq!(info.task_type, "t");
assert_eq!(info.status, TaskStatus::Pending);
assert!(info.started_at.is_none());
assert!(info.completed_at.is_none());
assert!(info.error.is_none());
}
#[test]
fn submit_with_some_params_creates_task() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager
.submit(&cx, "t", Some(serde_json::json!({"key": "value"})))
.unwrap();
assert!(manager.get_info(&id).is_some());
}
#[test]
fn fail_task_sets_error_result() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.fail_task(&id, "boom");
let result = manager.get_result(&id).unwrap();
assert!(!result.success);
assert_eq!(result.error, Some("boom".to_string()));
assert!(result.data.is_none());
}
#[test]
fn update_progress_nonexistent_does_not_panic() {
let manager = TaskManager::new_for_testing();
let fake_id = TaskId::from_string("nonexistent".to_string());
manager.update_progress(&fake_id, 0.5, None); }
#[test]
fn fail_task_on_completed_is_ignored() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.complete_task(&id, serde_json::json!({"done": true}));
manager.fail_task(&id, "too late");
let info = manager.get_info(&id).unwrap();
assert_eq!(info.status, TaskStatus::Completed);
let result = manager.get_result(&id).unwrap();
assert!(result.success);
}
#[test]
fn complete_task_on_failed_is_ignored() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.fail_task(&id, "something broke");
manager.complete_task(&id, serde_json::json!({"late": true}));
let info = manager.get_info(&id).unwrap();
assert_eq!(info.status, TaskStatus::Failed);
let result = manager.get_result(&id).unwrap();
assert!(!result.success);
}
#[test]
fn register_handler_replaces_existing() {
let manager = TaskManager::new_for_testing();
manager.register_handler("t", |_cx, _params| async {
Ok(serde_json::json!({"v": 1}))
});
manager.register_handler("t", |_cx, _params| async {
Ok(serde_json::json!({"v": 2}))
});
let cx = Cx::for_testing();
let id = manager.submit(&cx, "t", None).unwrap();
assert!(manager.get_info(&id).is_some());
}
#[test]
fn transition_to_running_sets_started_at() {
let task_id = TaskId::from_string("ts-test".to_string());
let mut state = TaskState {
info: TaskInfo {
id: task_id,
task_type: "t".to_string(),
status: TaskStatus::Pending,
progress: None,
message: None,
created_at: String::new(),
started_at: None,
completed_at: None,
error: None,
},
cancel_requested: false,
result: None,
cx: Cx::for_testing(),
};
assert!(state.info.started_at.is_none());
assert!(transition_state(&mut state, TaskStatus::Running));
assert!(state.info.started_at.is_some());
}
#[test]
fn transition_to_completed_sets_completed_at() {
let task_id = TaskId::from_string("ts-test".to_string());
let mut state = TaskState {
info: TaskInfo {
id: task_id,
task_type: "t".to_string(),
status: TaskStatus::Running,
progress: None,
message: None,
created_at: String::new(),
started_at: Some("earlier".to_string()),
completed_at: None,
error: None,
},
cancel_requested: false,
result: None,
cx: Cx::for_testing(),
};
assert!(state.info.completed_at.is_none());
assert!(transition_state(&mut state, TaskStatus::Completed));
assert!(state.info.completed_at.is_some());
}
#[test]
fn transition_to_failed_sets_completed_at() {
let task_id = TaskId::from_string("ts-test".to_string());
let mut state = TaskState {
info: TaskInfo {
id: task_id,
task_type: "t".to_string(),
status: TaskStatus::Running,
progress: None,
message: None,
created_at: String::new(),
started_at: Some("earlier".to_string()),
completed_at: None,
error: None,
},
cancel_requested: false,
result: None,
cx: Cx::for_testing(),
};
assert!(transition_state(&mut state, TaskStatus::Failed));
assert!(state.info.completed_at.is_some());
}
#[test]
fn transition_to_cancelled_sets_completed_at() {
let task_id = TaskId::from_string("ts-test".to_string());
let mut state = TaskState {
info: TaskInfo {
id: task_id,
task_type: "t".to_string(),
status: TaskStatus::Running,
progress: None,
message: None,
created_at: String::new(),
started_at: Some("earlier".to_string()),
completed_at: None,
error: None,
},
cancel_requested: false,
result: None,
cx: Cx::for_testing(),
};
assert!(transition_state(&mut state, TaskStatus::Cancelled));
assert!(state.info.completed_at.is_some());
}
#[test]
fn transition_invalid_returns_false() {
let task_id = TaskId::from_string("ts-test".to_string());
let mut state = TaskState {
info: TaskInfo {
id: task_id,
task_type: "t".to_string(),
status: TaskStatus::Pending,
progress: None,
message: None,
created_at: String::new(),
started_at: None,
completed_at: None,
error: None,
},
cancel_requested: false,
result: None,
cx: Cx::for_testing(),
};
assert!(!transition_state(&mut state, TaskStatus::Completed));
assert_eq!(state.info.status, TaskStatus::Pending);
}
#[test]
fn task_status_snapshot_debug_and_clone() {
let task_id = TaskId::from_string("snap-test".to_string());
let state = TaskState {
info: TaskInfo {
id: task_id,
task_type: "t".to_string(),
status: TaskStatus::Running,
progress: Some(0.5),
message: Some("testing".to_string()),
created_at: "now".to_string(),
started_at: Some("now".to_string()),
completed_at: None,
error: None,
},
cancel_requested: false,
result: None,
cx: Cx::for_testing(),
};
let snapshot = TaskStatusSnapshot::from(&state);
let debug = format!("{:?}", snapshot);
assert!(debug.contains("TaskStatusSnapshot"));
let cloned = snapshot.clone();
assert_eq!(cloned.info.status, TaskStatus::Running);
assert!(cloned.result.is_none());
}
#[test]
fn cleanup_completed_removes_failed_and_cancelled() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id1 = manager.submit(&cx, "t", None).unwrap();
let id2 = manager.submit(&cx, "t", None).unwrap();
let id3 = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id1).unwrap();
manager.complete_task(&id1, serde_json::json!({}));
manager.start_task(&id2).unwrap();
manager.fail_task(&id2, "error");
manager.cancel(&id3, None).unwrap();
assert_eq!(manager.total_count(), 3);
manager.cleanup_completed(std::time::Duration::from_secs(0));
assert_eq!(manager.total_count(), 0);
}
#[test]
fn set_notification_sender_replaces_existing() {
let manager = TaskManager::new_for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let count1 = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let count2 = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let c1 = Arc::clone(&count1);
let sender1: TaskNotificationSender = Arc::new(move |_| {
c1.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
});
manager.set_notification_sender(sender1);
let cx = Cx::for_testing();
let _id1 = manager.submit(&cx, "t", None).unwrap();
assert!(count1.load(std::sync::atomic::Ordering::SeqCst) > 0);
let c2 = Arc::clone(&count2);
let sender2: TaskNotificationSender = Arc::new(move |_| {
c2.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
});
manager.set_notification_sender(sender2);
let _id2 = manager.submit(&cx, "t", None).unwrap();
assert!(count2.load(std::sync::atomic::Ordering::SeqCst) > 0);
}
#[test]
fn cancel_with_custom_reason() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
let info = manager.cancel(&id, Some("Timeout".to_string())).unwrap();
assert_eq!(info.error, Some("Timeout".to_string()));
let result = manager.get_result(&id).unwrap();
assert_eq!(result.error, Some("Timeout".to_string()));
}
#[test]
fn can_transition_self_is_false() {
assert!(!can_transition(TaskStatus::Pending, TaskStatus::Pending));
assert!(!can_transition(TaskStatus::Running, TaskStatus::Running));
assert!(!can_transition(
TaskStatus::Completed,
TaskStatus::Completed
));
assert!(!can_transition(TaskStatus::Failed, TaskStatus::Failed));
assert!(!can_transition(
TaskStatus::Cancelled,
TaskStatus::Cancelled
));
}
#[test]
fn transition_state_identity_pending_returns_true() {
let task_id = TaskId::from_string("identity-test".to_string());
let mut state = TaskState {
info: TaskInfo {
id: task_id,
task_type: "t".to_string(),
status: TaskStatus::Pending,
progress: None,
message: None,
created_at: String::new(),
started_at: None,
completed_at: None,
error: None,
},
cancel_requested: false,
result: None,
cx: Cx::for_testing(),
};
assert!(transition_state(&mut state, TaskStatus::Pending));
assert_eq!(state.info.status, TaskStatus::Pending);
}
#[test]
fn list_tasks_no_filter_returns_all() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id1 = manager.submit(&cx, "t", None).unwrap();
let _id2 = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id1).unwrap();
manager.complete_task(&id1, serde_json::json!({}));
let all = manager.list_tasks(None);
assert_eq!(all.len(), 2);
}
#[test]
fn cancel_notification_includes_error_and_result() {
let manager = TaskManager::new_for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let events: Arc<std::sync::Mutex<Vec<TaskStatusNotificationParams>>> =
Arc::new(std::sync::Mutex::new(Vec::new()));
let sender_events = Arc::clone(&events);
let sender: TaskNotificationSender = Arc::new(move |request| {
if request.method == "notifications/tasks/status" {
let params: TaskStatusNotificationParams = request
.params
.as_ref()
.and_then(|v| serde_json::from_value(v.clone()).ok())
.unwrap();
sender_events.lock().unwrap().push(params);
}
});
manager.set_notification_sender(sender);
let cx = Cx::for_testing();
let id = manager.submit(&cx, "t", None).unwrap();
manager.cancel(&id, Some("user abort".to_string())).unwrap();
let recorded = events.lock().unwrap().clone();
let last = recorded.last().unwrap();
assert_eq!(last.status, TaskStatus::Cancelled);
assert_eq!(last.error, Some("user abort".to_string()));
assert!(last.result.is_some());
let result = last.result.as_ref().unwrap();
assert!(!result.success);
}
#[test]
fn complete_task_sets_progress_to_one() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.update_progress(&id, 0.5, None);
manager.complete_task(&id, serde_json::json!({}));
let info = manager.get_info(&id).unwrap();
assert_eq!(info.progress, Some(1.0));
}
#[test]
fn cleanup_completed_keeps_terminal_without_completed_at() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.complete_task(&id, serde_json::json!({}));
{
let mut tasks = manager.tasks.write().unwrap();
tasks.get_mut(&id).unwrap().info.completed_at = None;
}
manager.cleanup_completed(std::time::Duration::from_secs(0));
assert_eq!(manager.total_count(), 1);
}
#[test]
fn cleanup_completed_keeps_terminal_with_unparseable_timestamp() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.complete_task(&id, serde_json::json!({}));
{
let mut tasks = manager.tasks.write().unwrap();
tasks.get_mut(&id).unwrap().info.completed_at = Some("not-a-date".to_string());
}
manager.cleanup_completed(std::time::Duration::from_secs(0));
assert_eq!(manager.total_count(), 1);
}
#[test]
fn debug_output_with_tasks_and_handlers() {
let manager = TaskManager::new_for_testing();
manager.register_handler("type_a", |_cx, _params| async { Ok(serde_json::json!({})) });
manager.register_handler("type_b", |_cx, _params| async { Ok(serde_json::json!({})) });
let cx = Cx::for_testing();
let _ = manager.submit(&cx, "type_a", None).unwrap();
let _ = manager.submit(&cx, "type_b", None).unwrap();
let debug = format!("{:?}", manager);
assert!(debug.contains("task_count: 2"));
assert!(debug.contains("handler_count: 2"));
}
#[test]
fn multiple_handler_types_independent() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("analyze", |_cx, _params| async {
Ok(serde_json::json!({"type": "analyze"}))
});
manager.register_handler("summarize", |_cx, _params| async {
Ok(serde_json::json!({"type": "summarize"}))
});
let id_a = manager.submit(&cx, "analyze", None).unwrap();
let id_s = manager.submit(&cx, "summarize", None).unwrap();
let info_a = manager.get_info(&id_a).unwrap();
let info_s = manager.get_info(&id_s).unwrap();
assert_eq!(info_a.task_type, "analyze");
assert_eq!(info_s.task_type, "summarize");
}
#[test]
fn list_tasks_filter_failed() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.fail_task(&id, "err");
assert_eq!(manager.list_tasks(Some(TaskStatus::Failed)).len(), 1);
assert_eq!(manager.list_tasks(Some(TaskStatus::Completed)).len(), 0);
}
#[test]
fn list_tasks_filter_cancelled() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.cancel(&id, None).unwrap();
assert_eq!(manager.list_tasks(Some(TaskStatus::Cancelled)).len(), 1);
assert_eq!(manager.list_tasks(Some(TaskStatus::Pending)).len(), 0);
}
#[test]
fn progress_notification_includes_message() {
let manager = TaskManager::new_for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let events: Arc<std::sync::Mutex<Vec<TaskStatusNotificationParams>>> =
Arc::new(std::sync::Mutex::new(Vec::new()));
let sender_events = Arc::clone(&events);
let sender: TaskNotificationSender = Arc::new(move |request| {
if request.method == "notifications/tasks/status" {
let params: TaskStatusNotificationParams = request
.params
.as_ref()
.and_then(|v| serde_json::from_value(v.clone()).ok())
.unwrap();
sender_events.lock().unwrap().push(params);
}
});
manager.set_notification_sender(sender);
let cx = Cx::for_testing();
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.update_progress(&id, 0.75, Some("three quarters".to_string()));
let recorded = events.lock().unwrap().clone();
let progress_event = recorded
.iter()
.find(|e| e.progress == Some(0.75))
.expect("progress notification");
assert_eq!(progress_event.message, Some("three quarters".to_string()));
assert_eq!(progress_event.status, TaskStatus::Running);
}
#[test]
fn task_status_snapshot_includes_result() {
let task_id = TaskId::from_string("snap-result");
let state = TaskState {
info: TaskInfo {
id: task_id.clone(),
task_type: "t".to_string(),
status: TaskStatus::Completed,
progress: Some(1.0),
message: None,
created_at: "now".to_string(),
started_at: Some("now".to_string()),
completed_at: Some("now".to_string()),
error: None,
},
cancel_requested: false,
result: Some(TaskResult {
id: task_id,
success: true,
data: Some(serde_json::json!({"done": true})),
error: None,
}),
cx: Cx::for_testing(),
};
let snapshot = TaskStatusSnapshot::from(&state);
assert!(snapshot.result.is_some());
let result = snapshot.result.unwrap();
assert!(result.success);
assert_eq!(result.data, Some(serde_json::json!({"done": true})));
}
#[test]
fn submit_unknown_task_type_error_message() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
let err = manager.submit(&cx, "nonexistent_type", None).unwrap_err();
assert!(err.message.contains("Unknown task type"));
assert!(err.message.contains("nonexistent_type"));
}
#[test]
fn cancel_result_has_no_data() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.cancel(&id, Some("abort".to_string())).unwrap();
let result = manager.get_result(&id).unwrap();
assert!(!result.success);
assert!(result.data.is_none());
assert_eq!(result.error, Some("abort".to_string()));
}
#[test]
fn cancel_completed_task_returns_error() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.complete_task(&id, serde_json::json!({}));
let err = manager.cancel(&id, None).unwrap_err();
assert!(err.message.contains("terminal"));
}
#[test]
fn cancel_failed_task_returns_error() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.fail_task(&id, "broke");
let err = manager.cancel(&id, None).unwrap_err();
assert!(err.message.contains("terminal"));
}
#[test]
fn fail_task_on_pending_records_failure() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.fail_task(&id, "too early");
let info = manager.get_info(&id).unwrap();
assert_eq!(info.status, TaskStatus::Failed);
assert_eq!(info.error.as_deref(), Some("too early"));
assert!(info.completed_at.is_some());
let result = manager
.get_result(&id)
.expect("failed task should record a result");
assert!(!result.success);
assert_eq!(result.error.as_deref(), Some("too early"));
}
#[test]
fn spawn_task_skips_handler_for_pre_failed_pending_task() {
let manager = TaskManager::new();
let task_runs = Arc::new(AtomicU64::new(0));
let task_type = "never-run".to_string();
let task_id = TaskId::from_string("task-prefailed");
let task_cx = Cx::for_request_with_budget(Budget::INFINITE);
let now = chrono::Utc::now().to_rfc3339();
manager.register_handler(task_type.clone(), {
let task_runs = Arc::clone(&task_runs);
move |_cx, _params| {
let task_runs = Arc::clone(&task_runs);
async move {
task_runs.fetch_add(1, Ordering::SeqCst);
Ok(serde_json::json!({"unexpected": true}))
}
}
});
{
let mut tasks = manager.tasks.write().unwrap_or_else(|poisoned| {
warn!(target: targets::SERVER, "tasks lock poisoned in test, recovering");
poisoned.into_inner()
});
tasks.insert(
task_id.clone(),
TaskState {
info: TaskInfo {
id: task_id.clone(),
task_type: task_type.clone(),
status: TaskStatus::Failed,
progress: None,
message: None,
created_at: now,
started_at: None,
completed_at: Some(chrono::Utc::now().to_rfc3339()),
error: Some("prefailed".to_string()),
},
cancel_requested: false,
result: Some(TaskResult {
id: task_id.clone(),
success: false,
data: None,
error: Some("prefailed".to_string()),
}),
cx: task_cx.clone(),
},
);
}
manager.spawn_task(task_id.clone(), task_type, task_cx, serde_json::json!({}));
let deadline = std::time::Instant::now() + Duration::from_secs(1);
while std::time::Instant::now() < deadline {
if task_runs.load(Ordering::SeqCst) > 0 {
break;
}
thread::sleep(Duration::from_millis(10));
}
assert_eq!(
task_runs.load(Ordering::SeqCst),
0,
"pre-failed pending task must not execute its handler"
);
let info = manager
.get_info(&task_id)
.expect("prefailed task should remain present");
assert_eq!(info.status, TaskStatus::Failed);
assert_eq!(info.error.as_deref(), Some("prefailed"));
}
#[test]
fn complete_task_on_cancelled_is_ignored() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.cancel(&id, Some("aborted".to_string())).unwrap();
manager.complete_task(&id, serde_json::json!({"late": true}));
let info = manager.get_info(&id).unwrap();
assert_eq!(info.status, TaskStatus::Cancelled);
}
#[test]
fn update_progress_none_message_clears_previous() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.update_progress(&id, 0.3, Some("step 1".to_string()));
assert_eq!(
manager.get_info(&id).unwrap().message,
Some("step 1".to_string())
);
manager.update_progress(&id, 0.6, None);
assert!(manager.get_info(&id).unwrap().message.is_none());
}
#[test]
fn no_notification_sender_does_not_panic() {
let manager = TaskManager::new_for_testing();
let cx = Cx::for_testing();
manager.register_handler("t", |_cx, _params| async { Ok(serde_json::json!({})) });
let id = manager.submit(&cx, "t", None).unwrap();
manager.start_task(&id).unwrap();
manager.update_progress(&id, 0.5, None);
manager.complete_task(&id, serde_json::json!({}));
assert_eq!(manager.get_info(&id).unwrap().status, TaskStatus::Completed);
}
fn official_task_lifecycle() -> OfficialTaskLifecycle {
OfficialTaskLifecycle::new(
OfficialTaskLifecycleConfig::new(60_000, Some(5_000), 8)
.expect("valid bounded lifecycle configuration"),
)
}
fn task_input_request() -> OfficialTaskInputRequest {
OfficialTaskInputRequest {
method: OfficialTaskInputMethod::ElicitationCreate,
params: serde_json::json!({"message": "Approve the operation?"}),
}
}
fn final_tool_result() -> serde_json::Value {
serde_json::json!({
"resultType": "complete",
"content": [{"type": "text", "text": "done"}],
})
}
#[test]
fn task_02_a_positive() {
let lifecycle = official_task_lifecycle();
assert_eq!(lifecycle.storage_kind(), TaskStorageKind::ProcessLocal);
let created = lifecycle
.create(None)
.expect("create immediately readable task");
assert_eq!(created.status, OfficialTaskStatus::Working);
assert_eq!(created.ttl_ms, 60_000);
assert_eq!(created.poll_interval_ms, Some(5_000));
assert_eq!(created.task_id.as_str().len(), 43);
assert!(
created
.task_id
.as_str()
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')),
"local task IDs must use canonical unpadded base64url"
);
assert_eq!(
lifecycle
.get(&created.task_id)
.expect("created task lookup"),
created
);
let mut requests = BTreeMap::new();
requests.insert("approval".to_string(), task_input_request());
requests.insert("details".to_string(), task_input_request());
let waiting = lifecycle
.require_input(&created.task_id, requests, None)
.expect("working task enters input_required");
assert_eq!(waiting.status, OfficialTaskStatus::InputRequired);
assert_eq!(waiting.input_requests.as_ref().map(BTreeMap::len), Some(2));
let mut first_response = BTreeMap::new();
first_response.insert(
"approval".to_string(),
serde_json::json!({"approved": true}),
);
assert_eq!(
lifecycle
.update_input(&created.task_id, first_response)
.expect("partial update"),
OfficialTaskInputUpdate::Applied
);
let partially_satisfied = lifecycle.get(&created.task_id).expect("task lookup");
assert_eq!(
partially_satisfied.status,
OfficialTaskStatus::InputRequired
);
assert_eq!(
partially_satisfied
.input_requests
.as_ref()
.map(BTreeMap::len),
Some(1)
);
let mut final_response = BTreeMap::new();
final_response.insert("details".to_string(), serde_json::json!({"accepted": true}));
assert_eq!(
lifecycle
.update_input(&created.task_id, final_response)
.expect("final input update"),
OfficialTaskInputUpdate::Applied
);
assert_eq!(
lifecycle
.get(&created.task_id)
.expect("resumed task")
.status,
OfficialTaskStatus::Working
);
let completed = lifecycle
.complete(
&created.task_id,
final_tool_result(),
Some("Completed".to_string()),
)
.expect("complete after all input is satisfied");
assert_eq!(completed.status, OfficialTaskStatus::Completed);
assert_eq!(completed.result, Some(final_tool_result()));
assert!(completed.input_requests.is_none());
assert!(completed.error.is_none());
let failed = lifecycle.create(None).expect("create task to fail");
let failed = lifecycle
.fail(
&failed.task_id,
serde_json::json!({"code": -32603, "message": "Execution failed"}),
None,
)
.expect("working task records a JSON-RPC failure");
assert_eq!(failed.status, OfficialTaskStatus::Failed);
assert_eq!(
failed.status_message.as_deref(),
Some("Task execution failed"),
"the safe failure message is not copied from raw error data"
);
assert!(failed.result.is_none());
assert!(failed.error.is_some());
let cancelled = lifecycle.create(None).expect("create task to cancel");
lifecycle
.request_cancellation(&cancelled.task_id)
.expect("cooperative cancellation acknowledgement");
assert!(lifecycle.is_cancellation_requested(&cancelled.task_id));
let cancelled = lifecycle
.honor_cancellation(&cancelled.task_id, Some("Cancelled".to_string()))
.expect("supervised worker honors cancellation");
assert_eq!(cancelled.status, OfficialTaskStatus::Cancelled);
assert!(
lifecycle
.complete(&cancelled.task_id, final_tool_result(), None)
.is_err(),
"terminal task states are immutable"
);
assert_eq!(
lifecycle
.get(&cancelled.task_id)
.expect("cancelled task lookup")
.status,
OfficialTaskStatus::Cancelled
);
}
#[test]
fn task_02_a_planted_negative() {
let lifecycle = official_task_lifecycle();
let created = lifecycle.create(None).expect("create task");
let mut requests = BTreeMap::new();
requests.insert("approval".to_string(), task_input_request());
requests.insert("details".to_string(), task_input_request());
lifecycle
.require_input(&created.task_id, requests, None)
.expect("task awaits the same inputs as the positive case");
let before = serde_json::to_vec(
&lifecycle
.get(&created.task_id)
.expect("task snapshot before planted input"),
)
.expect("serialize stable snapshot");
let mut planted_unknown_response = BTreeMap::new();
planted_unknown_response.insert(
"not-approval".to_string(),
serde_json::json!({"approved": true}),
);
assert_eq!(
lifecycle
.update_input(&created.task_id, planted_unknown_response)
.expect("known task ignores an unknown input key"),
OfficialTaskInputUpdate::Ignored
);
let after = serde_json::to_vec(
&lifecycle
.get(&created.task_id)
.expect("task snapshot after planted input"),
)
.expect("serialize stable snapshot");
assert_eq!(after, before, "unknown input must not mutate task state");
assert_eq!(
lifecycle
.get(&created.task_id)
.expect("task remains readable")
.status,
OfficialTaskStatus::InputRequired
);
}
struct IdleSupervisor;
impl ApplicationTaskSupervisor for IdleSupervisor {
fn resume<'a>(
&'a self,
_cx: &'a Cx,
_handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
Box::pin(async { Ok(()) })
}
}
fn hosted_runtime() -> (FinalTaskRuntime, TaskServiceHost) {
let runtime = FinalTaskRuntime::in_memory(
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid timing policy"),
Arc::new(|_| {}),
);
let host = TaskServiceHost::install(
&runtime,
Arc::new(IdleSupervisor),
DEFAULT_CONCURRENT_TASK_HANDOFFS,
)
.expect("install host");
(runtime, host)
}
fn hosting_runtime() -> asupersync::runtime::Runtime {
RuntimeBuilder::current_thread()
.blocking_threads(1, 4)
.build()
.expect("caller-owned runtime")
}
#[test]
fn hosted_task_service_is_ready_only_while_hosted_and_serves_again() {
let (runtime, host) = hosted_runtime();
assert!(runtime.has_installed_task_service());
assert!(
!runtime.is_task_service_ready(),
"installing is not readiness"
);
hosting_runtime().block_on(async {
let cx = Cx::current().expect("caller execution context");
for serve in 0..2 {
let hosted = host.start_ready(&cx).await.expect("hosted service ready");
assert!(
runtime.is_task_service_ready(),
"serve {serve}: ready while hosted"
);
hosted.settle(&cx).await.expect("hosted service settles");
assert!(!runtime.is_task_service_ready(), "serve {serve}: settled");
}
});
}
#[test]
fn a_second_start_while_hosted_is_refused_and_leaves_the_first_ready() {
let (runtime, host) = hosted_runtime();
hosting_runtime().block_on(async {
let cx = Cx::current().expect("caller execution context");
let hosted = host.start_ready(&cx).await.expect("hosted service ready");
assert!(
host.start(&cx).is_err(),
"one runner cannot be hosted twice"
);
assert!(runtime.is_task_service_ready());
hosted.settle(&cx).await.expect("hosted service settles");
assert!(!runtime.is_task_service_ready());
});
}
#[test]
fn blocking_serve_loops_host_and_settle_the_same_runner() {
let (runtime, host) = hosted_runtime();
hosting_runtime().block_on(async {
let cx = Cx::current().expect("caller execution context");
let observed = runtime.clone();
let mut serve = cx
.spawn_blocking(move |serve_cx| {
let hosted = host.start_ready_blocking(&serve_cx)?;
let ready = observed.is_task_service_ready();
hosted.settle_blocking()?;
Ok::<_, McpError>((ready, observed.is_task_service_ready()))
})
.expect("blocking serve admitted");
let (while_hosted, after_settle) = serve
.join(&cx)
.await
.expect("blocking serve joins")
.expect("blocking serve hosts its service");
assert!(while_hosted);
assert!(!after_settle);
});
}
#[test]
fn a_runtime_with_a_caller_installed_service_cannot_host_another() {
let runtime = FinalTaskRuntime::in_memory(
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid timing policy"),
Arc::new(|_| {}),
);
assert!(!runtime.has_installed_task_service());
let _caller_runner = runtime
.install_task_service(1, Arc::new(IdleSupervisor))
.expect("caller installs its own service");
assert!(runtime.has_installed_task_service());
assert!(
TaskServiceHost::install(
&runtime,
Arc::new(IdleSupervisor),
DEFAULT_CONCURRENT_TASK_HANDOFFS,
)
.is_err()
);
}
struct HoldingSupervisor {
entered: Arc<AtomicBool>,
}
impl ApplicationTaskSupervisor for HoldingSupervisor {
fn resume<'a>(
&'a self,
cx: &'a Cx,
_handoff: FinalTaskSupervisorHandoff,
) -> FinalTaskSupervisorFuture<'a> {
let entered = Arc::clone(&self.entered);
Box::pin(async move {
entered.store(true, AtomicOrdering::SeqCst);
loop {
cx.checkpoint()
.map_err(|error| McpError::internal_error(error.to_string()))?;
asupersync::time::sleep(cx.now(), StdDuration::from_millis(1)).await;
}
})
}
}
fn settle_held_supervisor(cancel_first: bool) {
let runtime = FinalTaskRuntime::in_memory(
FinalTaskRuntimeConfig::new(60_000, Some(5_000)).expect("valid timing policy"),
Arc::new(|_| {}),
);
let entered = Arc::new(AtomicBool::new(false));
let host = TaskServiceHost::install(
&runtime,
Arc::new(HoldingSupervisor {
entered: Arc::clone(&entered),
}),
DEFAULT_CONCURRENT_TASK_HANDOFFS,
)
.expect("install host");
hosting_runtime().block_on(async {
let cx = Cx::current().expect("caller execution context");
let hosted = host.start_ready(&cx).await.expect("hosted service ready");
runtime
.create_task_with_work(final_test_work_descriptor(), None)
.expect("a hosted ready service admits creation");
for _ in 0..2_000 {
if entered.load(AtomicOrdering::SeqCst) {
break;
}
asupersync::time::sleep(cx.now(), StdDuration::from_millis(1)).await;
}
assert!(
entered.load(AtomicOrdering::SeqCst),
"the supervisor holds the handoff"
);
if cancel_first {
cx.cancel_with(CancelKind::User, Some("serve ended"));
}
let started = std::time::Instant::now();
hosted
.settle(&cx)
.await
.expect("settling ends the held supervisor within its bound");
assert!(started.elapsed() < HOSTED_TASK_SERVICE_SETTLEMENT_BOUND);
assert!(!runtime.is_task_service_ready());
});
}
#[test]
fn settling_ends_supervisor_work_in_flight_within_its_bound() {
settle_held_supervisor(false);
}
#[test]
fn settling_under_a_cancelled_serve_context_waits_for_the_runner() {
settle_held_supervisor(true);
}
}