1use std::collections::{BTreeSet, HashMap};
218use std::fmt::Write as _;
219use std::io;
220use std::sync::{Arc, Condvar, Mutex, OnceLock, RwLock, Weak};
221use std::time::{Duration, Instant};
222
223use async_trait::async_trait;
224
225use crate::error::JsonRpcError;
226use crate::protocol::{CallToolResult, InputRequests, InputResponses, TaskObject, TaskStatus};
227
228#[derive(Clone, Debug, Default)]
240pub struct CancellationToken {
241 inner: tokio_util::sync::CancellationToken,
242}
243
244impl CancellationToken {
245 #[must_use]
251 pub fn new() -> Self {
252 Self::default()
253 }
254
255 #[must_use]
257 pub fn is_cancelled(&self) -> bool {
258 self.inner.is_cancelled()
259 }
260
261 pub fn cancel(&self) {
263 self.inner.cancel();
264 }
265
266 pub async fn cancelled(&self) {
270 self.inner.cancelled().await;
271 }
272}
273
274const DEFAULT_TASK_TTL: Duration = Duration::from_secs(5 * 60);
279
280const DEFAULT_TASK_CLEANUP_INTERVAL: Duration = Duration::from_secs(60);
282
283const DEFAULT_MAX_RETAINED_TASKS: usize = 1_024;
285
286const DEFAULT_MAX_TASK_PAYLOAD_BYTES: usize = 4 * 1024 * 1024;
288
289const DEFAULT_MAX_RETAINED_TASK_BYTES: usize = 64 * 1024 * 1024;
291
292const DEFAULT_POLL_INTERVAL_MS: u64 = 2_000;
294
295#[derive(Debug, Clone, Copy, PartialEq, Eq)]
311#[non_exhaustive]
312pub struct TaskRetentionLimits {
313 pub max_tasks: usize,
315 pub max_payload_bytes: usize,
317 pub max_retained_bytes: usize,
319}
320
321impl TaskRetentionLimits {
322 #[must_use]
324 pub fn new() -> Self {
325 Self::default()
326 }
327
328 #[must_use]
333 pub const fn unbounded() -> Self {
334 Self {
335 max_tasks: usize::MAX,
336 max_payload_bytes: usize::MAX,
337 max_retained_bytes: usize::MAX,
338 }
339 }
340
341 #[must_use]
343 pub const fn max_tasks(mut self, max: usize) -> Self {
344 self.max_tasks = max;
345 self
346 }
347
348 #[must_use]
350 pub const fn max_payload_bytes(mut self, max: usize) -> Self {
351 self.max_payload_bytes = max;
352 self
353 }
354
355 #[must_use]
357 pub const fn max_retained_bytes(mut self, max: usize) -> Self {
358 self.max_retained_bytes = max;
359 self
360 }
361}
362
363impl Default for TaskRetentionLimits {
364 fn default() -> Self {
365 Self {
366 max_tasks: DEFAULT_MAX_RETAINED_TASKS,
367 max_payload_bytes: DEFAULT_MAX_TASK_PAYLOAD_BYTES,
368 max_retained_bytes: DEFAULT_MAX_RETAINED_TASK_BYTES,
369 }
370 }
371}
372
373#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
380#[non_exhaustive]
381pub struct TaskStoreUsage {
382 pub task_count: usize,
384 pub retained_bytes: usize,
386 pub reserved_bytes: usize,
388}
389
390impl TaskStoreUsage {
391 #[must_use]
397 pub fn charged_bytes(self) -> usize {
398 self.retained_bytes.saturating_add(self.reserved_bytes)
399 }
400}
401
402#[derive(Debug, Clone, Copy, PartialEq, Eq)]
416pub struct MemoryTaskStoreConfig {
417 pub default_ttl: Duration,
419 pub cleanup_interval: Duration,
421}
422
423impl MemoryTaskStoreConfig {
424 #[must_use]
426 pub fn new() -> Self {
427 Self::default()
428 }
429
430 #[must_use]
432 pub fn default_ttl(mut self, ttl: Duration) -> Self {
433 self.default_ttl = ttl;
434 self
435 }
436
437 #[must_use]
442 pub fn cleanup_interval(mut self, interval: Duration) -> Self {
443 self.cleanup_interval = interval;
444 self
445 }
446}
447
448impl Default for MemoryTaskStoreConfig {
449 fn default() -> Self {
450 Self {
451 default_ttl: DEFAULT_TASK_TTL,
452 cleanup_interval: DEFAULT_TASK_CLEANUP_INTERVAL,
453 }
454 }
455}
456
457fn duration_millis_saturated(duration: Duration) -> u64 {
458 u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
459}
460
461#[derive(Debug, Clone)]
469pub struct Task {
470 pub id: String,
472 pub tool_name: String,
474 pub arguments: serde_json::Value,
476 pub status: TaskStatus,
478 pub created_at: Instant,
480 pub created_at_str: String,
482 pub last_updated_at_str: String,
484 pub ttl: u64,
486 pub poll_interval: u64,
488 pub status_message: Option<String>,
490 pub meta: Option<serde_json::Value>,
492 pub result: Option<CallToolResult>,
494 pub error: Option<JsonRpcError>,
500 pub owner: TaskOwner,
505 pub input_requests: InputRequests,
507 pub answered_input_keys: BTreeSet<String>,
509 pub input_responses: InputResponses,
516 pub superseded_input_keys: BTreeSet<String>,
519 pub cancellation_token: CancellationToken,
521 pub completed_at: Option<Instant>,
523 pub completion_notify: Arc<tokio::sync::Notify>,
525}
526
527impl Task {
528 fn new(
530 id: String,
531 tool_name: String,
532 arguments: serde_json::Value,
533 ttl: u64,
534 owner: TaskOwner,
535 ) -> Self {
536 let now_str = chrono_now_iso8601();
537 Self {
538 id,
539 tool_name,
540 arguments,
541 status: TaskStatus::Working,
542 created_at: Instant::now(),
543 created_at_str: now_str.clone(),
544 last_updated_at_str: now_str,
545 ttl,
546 poll_interval: DEFAULT_POLL_INTERVAL_MS,
547 status_message: Some("Task started".to_string()),
548 meta: None,
549 result: None,
550 error: None,
551 owner,
552 input_requests: InputRequests::new(),
553 answered_input_keys: BTreeSet::new(),
554 input_responses: InputResponses::new(),
555 superseded_input_keys: BTreeSet::new(),
556 cancellation_token: CancellationToken::new(),
557 completed_at: None,
558 completion_notify: Arc::new(tokio::sync::Notify::new()),
559 }
560 }
561
562 pub fn to_task_object(&self) -> TaskObject {
569 TaskObject {
570 task_id: self.id.clone(),
571 status: self.status,
572 status_message: self.status_message.clone(),
573 created_at: self.created_at_str.clone(),
574 last_updated_at: self.last_updated_at_str.clone(),
575 ttl: Some(self.ttl),
576 poll_interval: Some(self.poll_interval),
577 result: None,
578 error: None,
579 meta: self.meta.clone(),
580 }
581 }
582
583 pub fn is_expired(&self) -> bool {
590 self.is_expired_at(Instant::now())
591 }
592
593 fn expires_at(&self) -> Option<Instant> {
594 self.created_at.checked_add(Duration::from_millis(self.ttl))
595 }
596
597 fn is_expired_at(&self, now: Instant) -> bool {
598 self.expires_at().is_some_and(|deadline| now >= deadline)
599 }
600
601 pub fn outstanding_input_requests(&self) -> &InputRequests {
603 &self.input_requests
604 }
605
606 pub fn is_cancelled(&self) -> bool {
608 self.cancellation_token.is_cancelled()
609 }
610
611 fn fail_retention_limit(&mut self) {
613 self.arguments = serde_json::Value::Null;
614 self.status = TaskStatus::Failed;
615 self.status_message = Some(RETENTION_FAILURE_STATUS.to_string());
616 self.meta = None;
617 self.result = None;
618 self.error = Some(JsonRpcError::internal_error(RETENTION_FAILURE_MESSAGE));
619 self.input_requests = InputRequests::new();
620 self.answered_input_keys = BTreeSet::new();
621 self.input_responses = InputResponses::new();
622 self.superseded_input_keys = BTreeSet::new();
623 self.completed_at = Some(Instant::now());
624 self.last_updated_at_str = chrono_now_iso8601();
625 }
626}
627
628#[derive(Debug, Clone)]
633struct StoredTask {
634 task: Task,
635 expiry_signalled: bool,
636 retained_bytes: usize,
637 reserved_bytes: usize,
638}
639
640impl StoredTask {
641 fn new(task: Task, retained_bytes: usize, reserved_bytes: usize) -> Self {
642 Self {
643 task,
644 expiry_signalled: false,
645 retained_bytes,
646 reserved_bytes,
647 }
648 }
649
650 fn signal_expiry(&mut self) -> bool {
652 if self.expiry_signalled {
653 return false;
654 }
655 self.expiry_signalled = true;
656 self.task.cancellation_token.cancel();
657 self.task.completion_notify.notify_waiters();
658 true
659 }
660
661 fn scrub_expired_payload(&mut self) {
663 self.task.tool_name = String::new();
664 self.task.arguments = serde_json::Value::Null;
665 self.task.status_message = None;
666 self.task.meta = None;
667 self.task.result = None;
668 self.task.error = None;
669 self.task.input_requests = InputRequests::new();
670 self.task.answered_input_keys = BTreeSet::new();
671 self.task.input_responses = InputResponses::new();
672 self.task.superseded_input_keys = BTreeSet::new();
673 self.reserved_bytes = 0;
674 }
675}
676
677impl std::ops::Deref for StoredTask {
678 type Target = Task;
679
680 fn deref(&self) -> &Self::Target {
681 &self.task
682 }
683}
684
685impl std::ops::DerefMut for StoredTask {
686 fn deref_mut(&mut self) -> &mut Self::Target {
687 &mut self.task
688 }
689}
690
691fn charged_bytes(retained: usize, reserved: usize, limit: usize) -> Result<usize> {
692 retained
693 .checked_add(reserved)
694 .filter(|charged| *charged <= limit)
695 .ok_or(TaskStoreError::RetentionLimitExceeded {
696 kind: TaskRetentionLimitKind::AggregateBytes,
697 limit,
698 })
699}
700
701fn prepare_stored_task(task: Task, limits: TaskRetentionLimits) -> Result<StoredTask> {
702 let retained_bytes = retained_payload_size(&task, limits.max_retained_bytes)?;
703 let reserved_bytes = if task.status.is_terminal() {
704 0
705 } else {
706 let mut fallback = task.clone();
710 fallback.fail_retention_limit();
711 let fallback_bytes = retained_payload_size(&fallback, limits.max_retained_bytes)?;
712 fallback_bytes.saturating_sub(retained_bytes)
713 };
714 charged_bytes(retained_bytes, reserved_bytes, limits.max_retained_bytes)?;
715 Ok(StoredTask::new(task, retained_bytes, reserved_bytes))
716}
717
718fn replace_stored_task(
719 data: &mut MemoryTaskStoreData,
720 task_id: &str,
721 replacement: StoredTask,
722 limit: usize,
723) -> Result<bool> {
724 let Some(current) = data.tasks.get(task_id) else {
725 return Ok(false);
726 };
727 let (retained_bytes, reserved_bytes) = replacement_totals(
728 data,
729 current.retained_bytes,
730 current.reserved_bytes,
731 replacement.retained_bytes,
732 replacement.reserved_bytes,
733 limit,
734 )?;
735
736 data.retained_bytes = retained_bytes;
737 data.reserved_bytes = reserved_bytes;
738 data.tasks.insert(task_id.to_string(), replacement);
739 Ok(true)
740}
741
742fn replacement_totals(
743 data: &MemoryTaskStoreData,
744 old_retained: usize,
745 old_reserved: usize,
746 new_retained: usize,
747 new_reserved: usize,
748 limit: usize,
749) -> Result<(usize, usize)> {
750 let retained_bytes = data
751 .retained_bytes
752 .checked_sub(old_retained)
753 .and_then(|bytes| bytes.checked_add(new_retained))
754 .ok_or(TaskStoreError::RetentionLimitExceeded {
755 kind: TaskRetentionLimitKind::AggregateBytes,
756 limit,
757 })?;
758 let reserved_bytes = data
759 .reserved_bytes
760 .checked_sub(old_reserved)
761 .and_then(|bytes| bytes.checked_add(new_reserved))
762 .ok_or(TaskStoreError::RetentionLimitExceeded {
763 kind: TaskRetentionLimitKind::AggregateBytes,
764 limit,
765 })?;
766 charged_bytes(retained_bytes, reserved_bytes, limit)?;
767 Ok((retained_bytes, reserved_bytes))
768}
769
770fn commit_removed_task(
771 data: &mut MemoryTaskStoreData,
772 task_id: &str,
773 replacement: StoredTask,
774 old_retained: usize,
775 old_reserved: usize,
776 limit: usize,
777) -> Result<()> {
778 let (retained_bytes, reserved_bytes) = replacement_totals(
779 data,
780 old_retained,
781 old_reserved,
782 replacement.retained_bytes,
783 replacement.reserved_bytes,
784 limit,
785 )?;
786 data.retained_bytes = retained_bytes;
787 data.reserved_bytes = reserved_bytes;
788 data.tasks.insert(task_id.to_string(), replacement);
789 Ok(())
790}
791
792fn commit_retention_failure(
793 data: &mut MemoryTaskStoreData,
794 task_id: &str,
795 mut task: StoredTask,
796 old_retained: usize,
797 old_reserved: usize,
798 limits: TaskRetentionLimits,
799) {
800 task.task.fail_retention_limit();
801 task.retained_bytes = retained_payload_size(&task.task, limits.max_retained_bytes)
802 .expect("reserved retention failure must fit the aggregate byte limit");
803 task.reserved_bytes = 0;
804 let notify = task.completion_notify.clone();
805 commit_removed_task(
806 data,
807 task_id,
808 task,
809 old_retained,
810 old_reserved,
811 limits.max_retained_bytes,
812 )
813 .expect("reserved retention failure must fit global task-store accounting");
814 notify.notify_waiters();
815}
816
817fn cancel_with_bounded_status(task: &mut Task, status: &'static str, scrub: bool) {
818 if scrub {
819 task.arguments = serde_json::Value::Null;
820 task.meta = None;
821 task.result = None;
822 task.error = None;
823 task.answered_input_keys = BTreeSet::new();
824 task.input_responses = InputResponses::new();
825 task.superseded_input_keys = BTreeSet::new();
826 }
827 task.input_requests = InputRequests::new();
828 task.status = TaskStatus::Cancelled;
829 task.status_message = Some(status.to_string());
830 task.completed_at = Some(Instant::now());
831 task.last_updated_at_str = chrono_now_iso8601();
832}
833
834pub fn generate_task_id() -> String {
847 let mut bytes = [0u8; 16];
848 getrandom::fill(&mut bytes).expect("system entropy source unavailable for task ID generation");
849 let mut id = String::with_capacity(2 * bytes.len());
850 for byte in bytes {
851 let _ = write!(id, "{byte:02x}");
852 }
853 id
854}
855
856pub type TaskOwner = Option<String>;
865
866pub fn owner_matches(owner: &TaskOwner, principal: Option<&str>) -> bool {
890 owner.as_deref() == principal
891}
892
893#[derive(Debug, Clone, Default, PartialEq, Eq)]
902pub struct AppliedInputResponses {
903 pub accepted: BTreeSet<String>,
905 pub ignored: BTreeSet<String>,
908 pub still_outstanding: BTreeSet<String>,
910}
911
912#[derive(Debug, Clone)]
922#[non_exhaustive]
923pub struct TaskResumeContext {
924 pub tool_name: String,
926 pub arguments: serde_json::Value,
928 pub input_responses: InputResponses,
930 pub cancellation_token: Option<CancellationToken>,
940}
941
942impl TaskResumeContext {
943 #[must_use]
967 pub fn new(
968 tool_name: impl Into<String>,
969 arguments: serde_json::Value,
970 input_responses: InputResponses,
971 ) -> Self {
972 Self {
973 tool_name: tool_name.into(),
974 arguments,
975 input_responses,
976 cancellation_token: None,
977 }
978 }
979
980 #[must_use]
987 pub fn with_cancellation_token(mut self, token: CancellationToken) -> Self {
988 self.cancellation_token = Some(token);
989 self
990 }
991}
992
993impl AppliedInputResponses {
994 pub fn is_complete(&self) -> bool {
996 self.still_outstanding.is_empty()
997 }
998}
999
1000#[derive(Debug, Clone, PartialEq, Eq)]
1014#[non_exhaustive]
1015pub enum TaskPresence {
1016 Present {
1018 owner: TaskOwner,
1020 },
1021 Expired {
1026 owner: TaskOwner,
1028 },
1029 Missing,
1031}
1032
1033impl TaskPresence {
1034 pub fn owner(&self) -> Option<&TaskOwner> {
1036 match self {
1037 Self::Present { owner } | Self::Expired { owner } => Some(owner),
1038 Self::Missing => None,
1039 }
1040 }
1041
1042 pub fn is_known(&self) -> bool {
1044 !matches!(self, Self::Missing)
1045 }
1046}
1047
1048#[derive(Debug, thiserror::Error)]
1062#[non_exhaustive]
1063pub enum TaskStoreError {
1064 #[error("encode error: {0}")]
1066 Encode(String),
1067 #[error("decode error: {0}")]
1069 Decode(String),
1070 #[error("backend error: {0}")]
1072 Backend(String),
1073 #[error("invalid task transition: {0}")]
1079 InvalidTransition(String),
1080 #[error("task retention {kind} limit exceeded (maximum {limit})")]
1086 RetentionLimitExceeded {
1087 kind: TaskRetentionLimitKind,
1089 limit: usize,
1091 },
1092}
1093
1094#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1096#[non_exhaustive]
1097pub enum TaskRetentionLimitKind {
1098 TaskCount,
1100 PayloadBytes,
1102 AggregateBytes,
1104}
1105
1106impl std::fmt::Display for TaskRetentionLimitKind {
1107 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1108 f.write_str(match self {
1109 Self::TaskCount => "task-count",
1110 Self::PayloadBytes => "payload-bytes",
1111 Self::AggregateBytes => "aggregate-bytes",
1112 })
1113 }
1114}
1115
1116fn same_input_request(
1124 a: &crate::protocol::InputRequest,
1125 b: &crate::protocol::InputRequest,
1126) -> bool {
1127 match (serde_json::to_value(a), serde_json::to_value(b)) {
1128 (Ok(a), Ok(b)) => a == b,
1129 _ => false,
1130 }
1131}
1132
1133pub type Result<T> = std::result::Result<T, TaskStoreError>;
1135
1136const RETENTION_FAILURE_MESSAGE: &str = "Task payload exceeded configured retention limits";
1137const RETENTION_FAILURE_STATUS: &str = "Task failed: retention limit exceeded";
1138
1139struct CountingWriter {
1141 written: usize,
1142 limit: usize,
1143 exceeded: bool,
1144}
1145
1146impl CountingWriter {
1147 fn new(limit: usize) -> Self {
1148 Self {
1149 written: 0,
1150 limit,
1151 exceeded: false,
1152 }
1153 }
1154}
1155
1156impl io::Write for CountingWriter {
1157 fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
1158 let Some(next) = self.written.checked_add(bytes.len()) else {
1159 self.exceeded = true;
1160 return Err(io::Error::other("encoded task payload exceeds byte limit"));
1161 };
1162 if next > self.limit {
1163 self.exceeded = true;
1164 return Err(io::Error::other("encoded task payload exceeds byte limit"));
1165 }
1166 self.written = next;
1167 Ok(bytes.len())
1168 }
1169
1170 fn flush(&mut self) -> io::Result<()> {
1171 Ok(())
1172 }
1173}
1174
1175fn encoded_size<T: serde::Serialize + ?Sized>(
1176 value: &T,
1177 limit: usize,
1178 kind: TaskRetentionLimitKind,
1179) -> Result<usize> {
1180 let mut writer = CountingWriter::new(limit);
1181 match serde_json::to_writer(&mut writer, value) {
1182 Ok(()) => Ok(writer.written),
1183 Err(_) if writer.exceeded => Err(TaskStoreError::RetentionLimitExceeded { kind, limit }),
1184 Err(error) => Err(TaskStoreError::Encode(error.to_string())),
1185 }
1186}
1187
1188#[derive(serde::Serialize)]
1189struct RetainedTaskPayload<'a> {
1190 tool_name: &'a str,
1191 arguments: &'a serde_json::Value,
1192 status_message: &'a Option<String>,
1193 meta: &'a Option<serde_json::Value>,
1194 result: &'a Option<CallToolResult>,
1195 error: &'a Option<JsonRpcError>,
1196 owner: &'a TaskOwner,
1197 input_requests: &'a InputRequests,
1198 answered_input_keys: &'a BTreeSet<String>,
1199 input_responses: &'a InputResponses,
1200 superseded_input_keys: &'a BTreeSet<String>,
1201}
1202
1203fn retained_payload_size(task: &Task, limit: usize) -> Result<usize> {
1204 encoded_size(
1205 &RetainedTaskPayload {
1206 tool_name: &task.tool_name,
1207 arguments: &task.arguments,
1208 status_message: &task.status_message,
1209 meta: &task.meta,
1210 result: &task.result,
1211 error: &task.error,
1212 owner: &task.owner,
1213 input_requests: &task.input_requests,
1214 answered_input_keys: &task.answered_input_keys,
1215 input_responses: &task.input_responses,
1216 superseded_input_keys: &task.superseded_input_keys,
1217 },
1218 limit,
1219 TaskRetentionLimitKind::AggregateBytes,
1220 )
1221}
1222
1223fn validate_payload<T: serde::Serialize + ?Sized>(
1224 payload: &T,
1225 limits: TaskRetentionLimits,
1226) -> Result<usize> {
1227 let (limit, kind) = if limits.max_payload_bytes <= limits.max_retained_bytes {
1228 (
1229 limits.max_payload_bytes,
1230 TaskRetentionLimitKind::PayloadBytes,
1231 )
1232 } else {
1233 (
1234 limits.max_retained_bytes,
1235 TaskRetentionLimitKind::AggregateBytes,
1236 )
1237 };
1238 encoded_size(payload, limit, kind)
1239}
1240
1241fn validate_prefixed_string(
1242 value: &str,
1243 prefix: &str,
1244 limits: TaskRetentionLimits,
1245) -> Result<usize> {
1246 fn check(
1247 value: &str,
1248 prefix: &str,
1249 limit: usize,
1250 kind: TaskRetentionLimitKind,
1251 ) -> Result<usize> {
1252 let Some(value_limit) = limit.checked_sub(prefix.len()) else {
1253 return Err(TaskStoreError::RetentionLimitExceeded { kind, limit });
1254 };
1255 encoded_size(value, value_limit, kind)?
1256 .checked_add(prefix.len())
1257 .ok_or(TaskStoreError::RetentionLimitExceeded { kind, limit })
1258 }
1259
1260 let (limit, kind) = if limits.max_payload_bytes <= limits.max_retained_bytes {
1261 (
1262 limits.max_payload_bytes,
1263 TaskRetentionLimitKind::PayloadBytes,
1264 )
1265 } else {
1266 (
1267 limits.max_retained_bytes,
1268 TaskRetentionLimitKind::AggregateBytes,
1269 )
1270 };
1271 check(value, prefix, limit, kind)
1272}
1273
1274pub type TaskSnapshot = (TaskObject, Option<CallToolResult>, Option<JsonRpcError>);
1280
1281#[async_trait]
1445pub trait TaskStore: Send + Sync + 'static {
1446 async fn create_task(
1456 &self,
1457 tool_name: &str,
1458 arguments: serde_json::Value,
1459 ttl: Option<u64>,
1460 owner: TaskOwner,
1461 ) -> Result<(String, CancellationToken)>;
1462
1463 async fn task_owner(&self, task_id: &str) -> Result<Option<TaskOwner>>;
1469
1470 async fn get_task(&self, task_id: &str) -> Result<Option<TaskObject>>;
1472
1473 async fn set_task_meta(&self, task_id: &str, meta: serde_json::Value) -> Result<bool> {
1478 let _ = (task_id, meta);
1479 Ok(false)
1480 }
1481
1482 async fn discard_task(&self, task_id: &str) -> Result<bool> {
1487 let _ = task_id;
1488 Ok(false)
1489 }
1490
1491 async fn get_task_result(&self, task_id: &str) -> Result<Option<TaskSnapshot>>;
1493
1494 async fn wait_for_completion(&self, task_id: &str) -> Result<Option<TaskSnapshot>>;
1500
1501 async fn list_tasks(&self, status_filter: Option<TaskStatus>) -> Result<Vec<TaskObject>>;
1503
1504 async fn require_input(
1600 &self,
1601 task_id: &str,
1602 requests: InputRequests,
1603 message: Option<&str>,
1604 ) -> Result<bool>;
1605
1606 async fn outstanding_input_requests(&self, task_id: &str) -> Result<Option<InputRequests>>;
1611
1612 async fn apply_input_responses(
1714 &self,
1715 task_id: &str,
1716 responses: InputResponses,
1717 ) -> Result<Option<AppliedInputResponses>>;
1718
1719 async fn task_presence(&self, task_id: &str) -> Result<TaskPresence> {
1743 Ok(match self.task_owner(task_id).await? {
1744 Some(owner) => TaskPresence::Present { owner },
1745 None => TaskPresence::Missing,
1746 })
1747 }
1748
1749 async fn input_responses(&self, task_id: &str) -> Result<Option<InputResponses>> {
1756 Ok(self
1757 .resume_context(task_id)
1758 .await?
1759 .map(|resume| resume.input_responses))
1760 }
1761
1762 async fn set_status(
1769 &self,
1770 task_id: &str,
1771 status: TaskStatus,
1772 message: Option<&str>,
1773 ) -> Result<bool> {
1774 let _ = (task_id, status, message);
1775 Ok(false)
1776 }
1777
1778 async fn resume_context(&self, task_id: &str) -> Result<Option<TaskResumeContext>> {
1856 let _ = task_id;
1857 Ok(None)
1858 }
1859
1860 async fn set_ttl(&self, task_id: &str, ttl_ms: u64) -> Result<bool>;
1865
1866 async fn complete_task(&self, task_id: &str, result: CallToolResult) -> Result<bool>;
1912
1913 async fn fail_task(&self, task_id: &str, error: JsonRpcError) -> Result<bool>;
1960
1961 async fn cancel_task(&self, task_id: &str, reason: Option<&str>) -> Result<Option<TaskObject>>;
2005}
2006
2007#[derive(Debug, Default)]
2008struct WorkerSignalState {
2009 generation: u64,
2010 shutdown: bool,
2011}
2012
2013#[derive(Debug, Default)]
2020struct WorkerSignal {
2021 state: Mutex<WorkerSignalState>,
2022 changed: Condvar,
2023}
2024
2025impl WorkerSignal {
2026 fn generation(&self) -> Option<u64> {
2027 let state = self.state.lock().ok()?;
2028 (!state.shutdown).then_some(state.generation)
2029 }
2030
2031 fn wake(&self) {
2032 if let Ok(mut state) = self.state.lock() {
2033 state.generation = state.generation.wrapping_add(1);
2034 self.changed.notify_one();
2035 }
2036 }
2037
2038 fn shutdown(&self) {
2039 if let Ok(mut state) = self.state.lock() {
2040 state.shutdown = true;
2041 state.generation = state.generation.wrapping_add(1);
2042 self.changed.notify_all();
2043 }
2044 }
2045
2046 fn wait_for_change(&self, generation: u64, timeout: Duration) -> bool {
2048 let Ok(state) = self.state.lock() else {
2049 return true;
2050 };
2051 if state.shutdown {
2052 return true;
2053 }
2054 if state.generation != generation {
2055 return false;
2056 }
2057 match self.changed.wait_timeout(state, timeout) {
2058 Ok((state, _)) => state.shutdown,
2059 Err(_) => true,
2060 }
2061 }
2062}
2063
2064#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
2065struct Retirement {
2066 signalled: usize,
2067 removed: usize,
2068}
2069
2070#[derive(Debug, Default)]
2071struct MemoryTaskStoreData {
2072 tasks: HashMap<String, StoredTask>,
2073 retained_bytes: usize,
2074 reserved_bytes: usize,
2075}
2076
2077impl MemoryTaskStoreData {
2078 fn usage(&self) -> TaskStoreUsage {
2079 TaskStoreUsage {
2080 task_count: self.tasks.len(),
2081 retained_bytes: self.retained_bytes,
2082 reserved_bytes: self.reserved_bytes,
2083 }
2084 }
2085}
2086
2087#[derive(Debug)]
2088struct MemoryTaskStoreState {
2089 data: RwLock<MemoryTaskStoreData>,
2090 config: MemoryTaskStoreConfig,
2091 retention_limits: TaskRetentionLimits,
2092 worker_signal: Arc<WorkerSignal>,
2093 worker_started: OnceLock<std::result::Result<(), String>>,
2094}
2095
2096impl MemoryTaskStoreState {
2097 fn retire_expired(&self, remove: bool) -> Retirement {
2098 let Ok(mut data) = self.data.write() else {
2099 return Retirement::default();
2100 };
2101 let now = Instant::now();
2102 let mut retirement = Retirement::default();
2103 let mut retired_old_retained = 0usize;
2104 let mut retired_new_retained = 0usize;
2105 let mut retired_old_reserved = 0usize;
2106 for task in data.tasks.values_mut() {
2107 if task.is_expired_at(now) && task.signal_expiry() {
2108 retirement.signalled += 1;
2109 let old_retained = task.retained_bytes;
2110 let old_reserved = task.reserved_bytes;
2111 task.scrub_expired_payload();
2112 task.retained_bytes = retained_payload_size(&task.task, usize::MAX)
2113 .expect("built-in task payload serialization is infallible");
2114 retired_old_retained = retired_old_retained
2115 .checked_add(old_retained)
2116 .expect("task-store retained-byte accounting overflowed");
2117 retired_new_retained = retired_new_retained
2118 .checked_add(task.retained_bytes)
2119 .expect("task-store retained-byte accounting overflowed");
2120 retired_old_reserved = retired_old_reserved
2121 .checked_add(old_reserved)
2122 .expect("task-store reserved-byte accounting overflowed");
2123 }
2124 }
2125 data.retained_bytes = data
2126 .retained_bytes
2127 .checked_sub(retired_old_retained)
2128 .and_then(|bytes| bytes.checked_add(retired_new_retained))
2129 .expect("task-store retained-byte accounting invariant violated");
2130 data.reserved_bytes = data
2131 .reserved_bytes
2132 .checked_sub(retired_old_reserved)
2133 .expect("task-store reserved-byte accounting invariant violated");
2134 charged_bytes(
2135 data.retained_bytes,
2136 data.reserved_bytes,
2137 self.retention_limits.max_retained_bytes,
2138 )
2139 .expect("expiry scrubbing exceeded the task-store aggregate-byte invariant");
2140 if remove {
2141 let before = data.tasks.len();
2142 let mut removed_retained = 0usize;
2143 let mut removed_reserved = 0usize;
2144 data.tasks.retain(|_, task| {
2145 let keep = !task.is_expired_at(now);
2146 if !keep {
2147 removed_retained = removed_retained
2148 .checked_add(task.retained_bytes)
2149 .expect("task-store retained-byte accounting overflowed");
2150 removed_reserved = removed_reserved
2151 .checked_add(task.reserved_bytes)
2152 .expect("task-store reserved-byte accounting overflowed");
2153 }
2154 keep
2155 });
2156 data.retained_bytes = data
2157 .retained_bytes
2158 .checked_sub(removed_retained)
2159 .expect("task-store retained-byte accounting invariant violated");
2160 data.reserved_bytes = data
2161 .reserved_bytes
2162 .checked_sub(removed_reserved)
2163 .expect("task-store reserved-byte accounting invariant violated");
2164 retirement.removed = before - data.tasks.len();
2165 }
2166 retirement
2167 }
2168
2169 fn next_expiry(&self) -> Option<Instant> {
2170 self.data
2171 .read()
2172 .ok()?
2173 .tasks
2174 .values()
2175 .filter_map(|task| {
2176 (!task.expiry_signalled)
2177 .then(|| task.expires_at())
2178 .flatten()
2179 })
2180 .min()
2181 }
2182}
2183
2184impl Drop for MemoryTaskStoreState {
2185 fn drop(&mut self) {
2186 self.worker_signal.shutdown();
2187 }
2188}
2189
2190fn next_deadline(a: Option<Instant>, b: Option<Instant>) -> Option<Instant> {
2191 match (a, b) {
2192 (Some(a), Some(b)) => Some(a.min(b)),
2193 (Some(deadline), None) | (None, Some(deadline)) => Some(deadline),
2194 (None, None) => None,
2195 }
2196}
2197
2198fn memory_task_store_worker(state: Weak<MemoryTaskStoreState>, signal: Arc<WorkerSignal>) {
2199 const MAX_SLEEP: Duration = Duration::from_secs(60 * 60);
2200
2201 let Some(initial) = state.upgrade() else {
2202 return;
2203 };
2204 let cleanup_interval = if initial.config.cleanup_interval.is_zero() {
2205 Duration::from_millis(1)
2206 } else {
2207 initial.config.cleanup_interval
2208 };
2209 let mut cleanup_at = Instant::now().checked_add(cleanup_interval);
2210 drop(initial);
2211
2212 loop {
2213 let Some(state) = state.upgrade() else {
2214 break;
2215 };
2216 let Some(generation) = signal.generation() else {
2217 break;
2218 };
2219
2220 let now = Instant::now();
2221 if cleanup_at.is_some_and(|deadline| now >= deadline) {
2222 state.retire_expired(true);
2223 cleanup_at = Instant::now().checked_add(cleanup_interval);
2227 } else {
2228 state.retire_expired(false);
2230 }
2231
2232 let deadline = next_deadline(cleanup_at, state.next_expiry());
2233 let timeout = deadline
2234 .map(|deadline| deadline.saturating_duration_since(Instant::now()))
2235 .unwrap_or(MAX_SLEEP)
2236 .min(MAX_SLEEP);
2237 drop(state);
2238
2239 if signal.wait_for_change(generation, timeout) {
2243 break;
2244 }
2245 }
2246}
2247
2248#[derive(Debug, Clone)]
2262pub struct MemoryTaskStore {
2263 state: Arc<MemoryTaskStoreState>,
2264}
2265
2266impl Default for MemoryTaskStore {
2267 fn default() -> Self {
2268 Self::new()
2269 }
2270}
2271
2272impl MemoryTaskStore {
2273 pub fn new() -> Self {
2278 Self::with_config(MemoryTaskStoreConfig::default())
2279 }
2280
2281 pub fn with_config(config: MemoryTaskStoreConfig) -> Self {
2284 Self::with_config_and_retention(config, TaskRetentionLimits::default())
2285 }
2286
2287 pub fn with_retention_limits(retention_limits: TaskRetentionLimits) -> Self {
2290 Self::with_config_and_retention(MemoryTaskStoreConfig::default(), retention_limits)
2291 }
2292
2293 pub fn with_config_and_retention(
2295 config: MemoryTaskStoreConfig,
2296 retention_limits: TaskRetentionLimits,
2297 ) -> Self {
2298 Self {
2299 state: Arc::new(MemoryTaskStoreState {
2300 data: RwLock::new(MemoryTaskStoreData::default()),
2301 config,
2302 retention_limits,
2303 worker_signal: Arc::new(WorkerSignal::default()),
2304 worker_started: OnceLock::new(),
2305 }),
2306 }
2307 }
2308
2309 fn ensure_worker(&self) -> Result<()> {
2310 let weak = Arc::downgrade(&self.state);
2311 let signal = self.state.worker_signal.clone();
2312 match self.state.worker_started.get_or_init(|| {
2313 std::thread::Builder::new()
2314 .name("tower-mcp-task-expiry".to_string())
2315 .spawn(move || memory_task_store_worker(weak, signal))
2316 .map(drop)
2317 .map_err(|error| format!("failed to start task expiry worker: {error}"))
2318 }) {
2319 Ok(()) => Ok(()),
2320 Err(error) => Err(TaskStoreError::Backend(error.clone())),
2321 }
2322 }
2323
2324 pub fn cleanup_expired(&self) -> usize {
2360 self.state.retire_expired(true).removed
2361 }
2362
2363 #[must_use]
2368 pub fn usage(&self) -> TaskStoreUsage {
2369 match self.state.data.read() {
2370 Ok(data) => data.usage(),
2371 Err(poisoned) => poisoned.into_inner().usage(),
2372 }
2373 }
2374
2375 #[cfg(test)]
2377 pub fn len(&self) -> usize {
2378 if let Ok(data) = self.state.data.read() {
2379 data.tasks.len()
2380 } else {
2381 0
2382 }
2383 }
2384
2385 #[cfg(test)]
2387 pub fn is_empty(&self) -> bool {
2388 self.len() == 0
2389 }
2390}
2391
2392#[async_trait]
2393impl TaskStore for MemoryTaskStore {
2394 async fn create_task(
2395 &self,
2396 tool_name: &str,
2397 arguments: serde_json::Value,
2398 ttl: Option<u64>,
2399 owner: TaskOwner,
2400 ) -> Result<(String, CancellationToken)> {
2401 self.ensure_worker()?;
2402 let limits = self.state.retention_limits;
2403 validate_payload(tool_name, limits)?;
2404 validate_payload(&arguments, limits)?;
2405 validate_payload(&owner, limits)?;
2406 let id = generate_task_id();
2407 let ttl = ttl.unwrap_or_else(|| duration_millis_saturated(self.state.config.default_ttl));
2408 let task = Task::new(id.clone(), tool_name.to_string(), arguments, ttl, owner);
2409 let token = task.cancellation_token.clone();
2410 let stored = prepare_stored_task(task, limits)?;
2411
2412 self.state.retire_expired(true);
2416 let mut data = self.state.data.write().map_err(|_| {
2417 TaskStoreError::Backend("in-memory task store lock poisoned".to_string())
2418 })?;
2419 if data.tasks.len() >= limits.max_tasks {
2420 return Err(TaskStoreError::RetentionLimitExceeded {
2421 kind: TaskRetentionLimitKind::TaskCount,
2422 limit: limits.max_tasks,
2423 });
2424 }
2425 let retained_bytes = data
2426 .retained_bytes
2427 .checked_add(stored.retained_bytes)
2428 .ok_or(TaskStoreError::RetentionLimitExceeded {
2429 kind: TaskRetentionLimitKind::AggregateBytes,
2430 limit: limits.max_retained_bytes,
2431 })?;
2432 let reserved_bytes = data
2433 .reserved_bytes
2434 .checked_add(stored.reserved_bytes)
2435 .ok_or(TaskStoreError::RetentionLimitExceeded {
2436 kind: TaskRetentionLimitKind::AggregateBytes,
2437 limit: limits.max_retained_bytes,
2438 })?;
2439 charged_bytes(retained_bytes, reserved_bytes, limits.max_retained_bytes)?;
2440 data.retained_bytes = retained_bytes;
2441 data.reserved_bytes = reserved_bytes;
2442 data.tasks.insert(id.clone(), stored);
2443 drop(data);
2444 self.state.worker_signal.wake();
2445
2446 Ok((id, token))
2447 }
2448
2449 async fn get_task(&self, task_id: &str) -> Result<Option<TaskObject>> {
2450 self.state.retire_expired(false);
2451 Ok(if let Ok(data) = self.state.data.read() {
2452 data.tasks
2453 .get(task_id)
2454 .filter(|t| !t.is_expired())
2455 .map(|t| t.to_task_object())
2456 } else {
2457 None
2458 })
2459 }
2460
2461 async fn set_task_meta(&self, task_id: &str, meta: serde_json::Value) -> Result<bool> {
2462 self.state.retire_expired(false);
2463 let Ok(mut data) = self.state.data.write() else {
2464 return Ok(false);
2465 };
2466 let Some(current) = data.tasks.get(task_id).filter(|task| !task.is_expired()) else {
2467 return Ok(false);
2468 };
2469 let limits = self.state.retention_limits;
2470 validate_payload(&meta, limits)?;
2471 let mut task = current.task.clone();
2472 task.meta = Some(meta);
2473 let replacement = prepare_stored_task(task, limits)?;
2474 replace_stored_task(&mut data, task_id, replacement, limits.max_retained_bytes)
2475 }
2476
2477 async fn discard_task(&self, task_id: &str) -> Result<bool> {
2478 let Ok(mut data) = self.state.data.write() else {
2479 return Ok(false);
2480 };
2481 let Some(removed) = data.tasks.remove(task_id) else {
2482 return Ok(false);
2483 };
2484 data.retained_bytes = data
2485 .retained_bytes
2486 .checked_sub(removed.retained_bytes)
2487 .expect("task-store retained-byte accounting invariant violated");
2488 data.reserved_bytes = data
2489 .reserved_bytes
2490 .checked_sub(removed.reserved_bytes)
2491 .expect("task-store reserved-byte accounting invariant violated");
2492 Ok(true)
2493 }
2494
2495 async fn task_owner(&self, task_id: &str) -> Result<Option<TaskOwner>> {
2496 self.state.retire_expired(false);
2497 Ok(if let Ok(data) = self.state.data.read() {
2498 data.tasks
2499 .get(task_id)
2500 .filter(|t| !t.is_expired())
2501 .map(|t| t.owner.clone())
2502 } else {
2503 None
2504 })
2505 }
2506
2507 async fn task_presence(&self, task_id: &str) -> Result<TaskPresence> {
2512 self.state.retire_expired(false);
2513 let Ok(data) = self.state.data.read() else {
2514 return Ok(TaskPresence::Missing);
2515 };
2516 Ok(match data.tasks.get(task_id) {
2517 Some(task) if task.is_expired() => TaskPresence::Expired {
2518 owner: task.owner.clone(),
2519 },
2520 Some(task) => TaskPresence::Present {
2521 owner: task.owner.clone(),
2522 },
2523 None => TaskPresence::Missing,
2524 })
2525 }
2526
2527 async fn get_task_result(&self, task_id: &str) -> Result<Option<TaskSnapshot>> {
2528 self.state.retire_expired(false);
2529 Ok(if let Ok(data) = self.state.data.read() {
2530 data.tasks
2531 .get(task_id)
2532 .filter(|t| !t.is_expired())
2533 .map(|t| (t.to_task_object(), t.result.clone(), t.error.clone()))
2534 } else {
2535 None
2536 })
2537 }
2538
2539 async fn wait_for_completion(&self, task_id: &str) -> Result<Option<TaskSnapshot>> {
2540 self.state.retire_expired(false);
2541 let (mut notified, cancellation) = {
2544 let Ok(data) = self.state.data.read() else {
2545 return Ok(None);
2546 };
2547 let Some(task) = data.tasks.get(task_id).filter(|t| !t.is_expired()) else {
2548 return Ok(None);
2549 };
2550 if task.status.is_terminal() {
2551 return Ok(Some((
2552 task.to_task_object(),
2553 task.result.clone(),
2554 task.error.clone(),
2555 )));
2556 }
2557 let mut notified = Box::pin(task.completion_notify.clone().notified_owned());
2558 let _ = notified.as_mut().enable();
2559 (notified, task.cancellation_token.clone())
2560 };
2561
2562 let cancellation_fired = tokio::select! {
2563 _ = &mut notified => false,
2564 _ = cancellation.cancelled() => true,
2565 };
2566
2567 if cancellation_fired {
2568 match self.get_task_result(task_id).await? {
2569 Some((task, _, _)) if !task.status.is_terminal() => {
2576 notified.await;
2577 }
2578 snapshot => return Ok(snapshot),
2579 }
2580 }
2581
2582 self.get_task_result(task_id).await
2584 }
2585
2586 async fn list_tasks(&self, status_filter: Option<TaskStatus>) -> Result<Vec<TaskObject>> {
2587 self.state.retire_expired(false);
2588 Ok(if let Ok(data) = self.state.data.read() {
2589 data.tasks
2590 .values()
2591 .filter(|t| !t.is_expired())
2592 .filter(|t| status_filter.is_none() || status_filter == Some(t.status))
2593 .map(|t| t.to_task_object())
2594 .collect()
2595 } else {
2596 vec![]
2597 })
2598 }
2599
2600 async fn require_input(
2601 &self,
2602 task_id: &str,
2603 requests: InputRequests,
2604 message: Option<&str>,
2605 ) -> Result<bool> {
2606 self.state.retire_expired(false);
2607 let Ok(mut data) = self.state.data.write() else {
2608 return Ok(false);
2609 };
2610 let Some(current) = data.tasks.get(task_id).filter(|task| !task.is_expired()) else {
2611 return Ok(false);
2612 };
2613 if current.status.is_terminal() {
2614 return Ok(false);
2615 }
2616 let limits = self.state.retention_limits;
2617 validate_payload(&requests, limits)?;
2618 if let Some(message) = message {
2619 validate_payload(message, limits)?;
2620 }
2621 let mut task = current.task.clone();
2622
2623 let reused: Vec<String> = requests
2638 .iter()
2639 .filter(|(key, request)| {
2640 if task.answered_input_keys.contains(*key)
2641 || task.superseded_input_keys.contains(*key)
2642 {
2643 return true;
2644 }
2645 match task.input_requests.get(*key) {
2646 Some(current) => !same_input_request(current, request),
2647 None => false,
2648 }
2649 })
2650 .map(|(key, _)| key.clone())
2651 .collect();
2652 if !reused.is_empty() {
2653 return Err(TaskStoreError::InvalidTransition(format!(
2654 "input request keys must be unique over a task's lifetime, but {} \
2655 already {} used by this task; use a new key to ask again",
2656 reused.join(", "),
2657 if reused.len() == 1 { "was" } else { "were" },
2658 )));
2659 }
2660
2661 for key in std::mem::take(&mut task.input_requests).into_keys() {
2665 if !requests.contains_key(&key) {
2666 task.superseded_input_keys.insert(key);
2667 }
2668 }
2669
2670 task.input_requests = requests;
2671 task.status = TaskStatus::InputRequired;
2672 task.status_message = Some(
2673 message
2674 .map(str::to_string)
2675 .unwrap_or_else(|| "Awaiting client input".to_string()),
2676 );
2677 task.last_updated_at_str = chrono_now_iso8601();
2678 let replacement = prepare_stored_task(task, limits)?;
2679 replace_stored_task(&mut data, task_id, replacement, limits.max_retained_bytes)
2680 }
2681
2682 async fn outstanding_input_requests(&self, task_id: &str) -> Result<Option<InputRequests>> {
2683 self.state.retire_expired(false);
2684 Ok(if let Ok(data) = self.state.data.read() {
2685 data.tasks
2686 .get(task_id)
2687 .filter(|t| !t.is_expired())
2688 .map(|t| t.input_requests.clone())
2689 } else {
2690 None
2691 })
2692 }
2693
2694 async fn apply_input_responses(
2695 &self,
2696 task_id: &str,
2697 responses: InputResponses,
2698 ) -> Result<Option<AppliedInputResponses>> {
2699 self.state.retire_expired(false);
2700 let Ok(mut data) = self.state.data.write() else {
2701 return Ok(None);
2702 };
2703 let Some(current) = data.tasks.get(task_id).filter(|task| !task.is_expired()) else {
2704 return Ok(None);
2705 };
2706 if current.status.is_terminal() {
2707 return Ok(None);
2708 }
2709 let limits = self.state.retention_limits;
2710 let accepted_payload: std::collections::BTreeMap<&str, &crate::protocol::InputResponse> =
2711 responses
2712 .iter()
2713 .filter(|(key, _)| current.input_requests.contains_key(*key))
2714 .map(|(key, response)| (key.as_str(), response))
2715 .collect();
2716 if !accepted_payload.is_empty() {
2717 validate_payload(&accepted_payload, limits)?;
2718 }
2719 let mut task = current.task.clone();
2720
2721 let mut applied = AppliedInputResponses::default();
2722 for (key, response) in responses {
2723 if task.input_requests.remove(&key).is_some() {
2724 task.answered_input_keys.insert(key.clone());
2725 task.input_responses.insert(key.clone(), response);
2726 applied.accepted.insert(key);
2727 } else {
2728 applied.ignored.insert(key);
2732 }
2733 }
2734 applied.still_outstanding = task.input_requests.keys().cloned().collect();
2735
2736 if !applied.accepted.is_empty() {
2737 task.last_updated_at_str = chrono_now_iso8601();
2738 }
2739 if applied.is_complete() && task.status == TaskStatus::InputRequired {
2740 task.status = TaskStatus::Working;
2741 task.status_message = Some("Task resumed".to_string());
2742 }
2743
2744 let replacement = prepare_stored_task(task, limits)?;
2745 if replace_stored_task(&mut data, task_id, replacement, limits.max_retained_bytes)? {
2746 Ok(Some(applied))
2747 } else {
2748 Ok(None)
2749 }
2750 }
2751
2752 async fn set_status(
2753 &self,
2754 task_id: &str,
2755 status: TaskStatus,
2756 message: Option<&str>,
2757 ) -> Result<bool> {
2758 if status.is_terminal() {
2759 return Err(TaskStoreError::InvalidTransition(format!(
2760 "set_status is for non-terminal progress; use complete_task, fail_task, or cancel_task to reach {status:?}"
2761 )));
2762 }
2763 self.state.retire_expired(false);
2764 let Ok(mut data) = self.state.data.write() else {
2765 return Ok(false);
2766 };
2767 let Some(current) = data.tasks.get(task_id).filter(|task| !task.is_expired()) else {
2768 return Ok(false);
2769 };
2770 if current.status.is_terminal() {
2771 return Ok(false);
2772 }
2773 let limits = self.state.retention_limits;
2774 if let Some(message) = message {
2775 validate_payload(message, limits)?;
2776 }
2777 let mut task = current.task.clone();
2778 task.status = status;
2779 if let Some(message) = message {
2780 task.status_message = Some(message.to_string());
2781 }
2782 task.last_updated_at_str = chrono_now_iso8601();
2783 let replacement = prepare_stored_task(task, limits)?;
2784 replace_stored_task(&mut data, task_id, replacement, limits.max_retained_bytes)
2785 }
2786
2787 async fn resume_context(&self, task_id: &str) -> Result<Option<TaskResumeContext>> {
2788 self.state.retire_expired(false);
2789 let Ok(data) = self.state.data.read() else {
2790 return Ok(None);
2791 };
2792 Ok(data
2793 .tasks
2794 .get(task_id)
2795 .filter(|task| !task.is_expired())
2796 .map(|task| {
2797 TaskResumeContext::new(
2798 task.tool_name.clone(),
2799 task.arguments.clone(),
2800 task.input_responses.clone(),
2801 )
2802 .with_cancellation_token(task.cancellation_token.clone())
2803 }))
2804 }
2805
2806 async fn set_ttl(&self, task_id: &str, ttl_ms: u64) -> Result<bool> {
2807 let Ok(mut data) = self.state.data.write() else {
2808 return Ok(false);
2809 };
2810 let Some(task) = data.tasks.get_mut(task_id) else {
2811 return Ok(false);
2812 };
2813 if task.is_expired() {
2814 drop(data);
2815 self.state.retire_expired(false);
2816 return Ok(false);
2817 }
2818 task.ttl = ttl_ms;
2819 task.last_updated_at_str = chrono_now_iso8601();
2820 drop(data);
2821 self.state.retire_expired(false);
2822 self.state.worker_signal.wake();
2823 Ok(true)
2824 }
2825
2826 async fn complete_task(&self, task_id: &str, result: CallToolResult) -> Result<bool> {
2827 self.state.retire_expired(false);
2828 let Ok(mut data) = self.state.data.write() else {
2829 return Ok(false);
2830 };
2831 let Some(mut task) = data.tasks.remove(task_id) else {
2832 return Ok(false);
2833 };
2834 if task.status.is_terminal() {
2835 data.tasks.insert(task_id.to_string(), task);
2836 return Ok(false);
2837 }
2838 if task.is_expired() {
2839 data.tasks.insert(task_id.to_string(), task);
2840 drop(data);
2841 self.state.retire_expired(false);
2842 return Ok(false);
2843 }
2844 let limits = self.state.retention_limits;
2845 let old_retained = task.retained_bytes;
2846 let old_reserved = task.reserved_bytes;
2847 if let Err(error) = validate_payload(&result, limits) {
2848 drop(result);
2849 commit_retention_failure(&mut data, task_id, task, old_retained, old_reserved, limits);
2850 return Err(error);
2851 }
2852 task.status = TaskStatus::Completed;
2853 task.status_message = Some("Task completed".to_string());
2854 task.result = Some(result);
2855 task.input_requests = InputRequests::new();
2856 task.completed_at = Some(Instant::now());
2857 task.last_updated_at_str = chrono_now_iso8601();
2858 task.retained_bytes = match retained_payload_size(&task.task, limits.max_retained_bytes) {
2859 Ok(bytes) => bytes,
2860 Err(error) => {
2861 commit_retention_failure(
2862 &mut data,
2863 task_id,
2864 task,
2865 old_retained,
2866 old_reserved,
2867 limits,
2868 );
2869 return Err(error);
2870 }
2871 };
2872 task.reserved_bytes = 0;
2873 let (retained_bytes, reserved_bytes) = match replacement_totals(
2874 &data,
2875 old_retained,
2876 old_reserved,
2877 task.retained_bytes,
2878 0,
2879 limits.max_retained_bytes,
2880 ) {
2881 Ok(totals) => totals,
2882 Err(error) => {
2883 commit_retention_failure(
2884 &mut data,
2885 task_id,
2886 task,
2887 old_retained,
2888 old_reserved,
2889 limits,
2890 );
2891 return Err(error);
2892 }
2893 };
2894 let notify = task.completion_notify.clone();
2895 data.retained_bytes = retained_bytes;
2896 data.reserved_bytes = reserved_bytes;
2897 data.tasks.insert(task_id.to_string(), task);
2898 notify.notify_waiters();
2899 Ok(true)
2900 }
2901
2902 async fn fail_task(&self, task_id: &str, error: JsonRpcError) -> Result<bool> {
2903 self.state.retire_expired(false);
2904 let Ok(mut data) = self.state.data.write() else {
2905 return Ok(false);
2906 };
2907 let Some(mut task) = data.tasks.remove(task_id) else {
2908 return Ok(false);
2909 };
2910 if task.status.is_terminal() {
2911 data.tasks.insert(task_id.to_string(), task);
2912 return Ok(false);
2913 }
2914 if task.is_expired() {
2915 data.tasks.insert(task_id.to_string(), task);
2916 drop(data);
2917 self.state.retire_expired(false);
2918 return Ok(false);
2919 }
2920 let limits = self.state.retention_limits;
2921 let old_retained = task.retained_bytes;
2922 let old_reserved = task.reserved_bytes;
2923 let error_validation = validate_payload(&error, limits);
2924 if let Err(retention_error) = error_validation {
2925 drop(error);
2926 commit_retention_failure(&mut data, task_id, task, old_retained, old_reserved, limits);
2927 return Err(retention_error);
2928 }
2929 if let Err(retention_error) =
2930 validate_prefixed_string(&error.message, "Task failed: ", limits)
2931 {
2932 drop(error);
2933 commit_retention_failure(&mut data, task_id, task, old_retained, old_reserved, limits);
2934 return Err(retention_error);
2935 }
2936 let status_message = format!("Task failed: {}", error.message);
2937 task.status = TaskStatus::Failed;
2938 task.status_message = Some(status_message);
2939 task.error = Some(error);
2940 task.input_requests = InputRequests::new();
2941 task.completed_at = Some(Instant::now());
2942 task.last_updated_at_str = chrono_now_iso8601();
2943 task.retained_bytes = match retained_payload_size(&task.task, limits.max_retained_bytes) {
2944 Ok(bytes) => bytes,
2945 Err(error) => {
2946 commit_retention_failure(
2947 &mut data,
2948 task_id,
2949 task,
2950 old_retained,
2951 old_reserved,
2952 limits,
2953 );
2954 return Err(error);
2955 }
2956 };
2957 task.reserved_bytes = 0;
2958 let (retained_bytes, reserved_bytes) = match replacement_totals(
2959 &data,
2960 old_retained,
2961 old_reserved,
2962 task.retained_bytes,
2963 0,
2964 limits.max_retained_bytes,
2965 ) {
2966 Ok(totals) => totals,
2967 Err(error) => {
2968 commit_retention_failure(
2969 &mut data,
2970 task_id,
2971 task,
2972 old_retained,
2973 old_reserved,
2974 limits,
2975 );
2976 return Err(error);
2977 }
2978 };
2979 let notify = task.completion_notify.clone();
2980 data.retained_bytes = retained_bytes;
2981 data.reserved_bytes = reserved_bytes;
2982 data.tasks.insert(task_id.to_string(), task);
2983 notify.notify_waiters();
2984 Ok(true)
2985 }
2986
2987 async fn cancel_task(&self, task_id: &str, reason: Option<&str>) -> Result<Option<TaskObject>> {
2988 self.state.retire_expired(false);
2989 let Ok(mut data) = self.state.data.write() else {
2990 return Ok(None);
2991 };
2992 let Some(mut task) = data.tasks.remove(task_id) else {
2993 return Ok(None);
2994 };
2995 if task.is_expired() {
2996 data.tasks.insert(task_id.to_string(), task);
2997 drop(data);
2998 self.state.retire_expired(false);
2999 return Ok(None);
3000 }
3001
3002 task.cancellation_token.cancel();
3004
3005 if !task.status.is_terminal() {
3007 let limits = self.state.retention_limits;
3008 let old_retained = task.retained_bytes;
3009 let old_reserved = task.reserved_bytes;
3010 let bounded_reason = match reason {
3011 Some(reason) => validate_prefixed_string(reason, "Cancelled: ", limits).is_ok(),
3012 None => validate_payload("Task cancelled", limits).is_ok(),
3013 };
3014 if bounded_reason {
3015 let requested_status = reason
3016 .map(|reason| format!("Cancelled: {reason}"))
3017 .unwrap_or_else(|| "Task cancelled".to_string());
3018 task.input_requests = InputRequests::new();
3019 task.status = TaskStatus::Cancelled;
3020 task.status_message = Some(requested_status);
3021 task.completed_at = Some(Instant::now());
3022 task.last_updated_at_str = chrono_now_iso8601();
3023 } else {
3024 cancel_with_bounded_status(
3025 &mut task.task,
3026 "Task cancelled: retention limit exceeded",
3027 true,
3028 );
3029 }
3030
3031 let candidate_bytes = retained_payload_size(&task.task, limits.max_retained_bytes);
3032 let totals = candidate_bytes.and_then(|bytes| {
3033 replacement_totals(
3034 &data,
3035 old_retained,
3036 old_reserved,
3037 bytes,
3038 0,
3039 limits.max_retained_bytes,
3040 )
3041 .map(|totals| (bytes, totals))
3042 });
3043 let (new_retained, (retained_bytes, reserved_bytes)) = match totals {
3044 Ok(totals) => totals,
3045 Err(_) => {
3046 cancel_with_bounded_status(
3047 &mut task.task,
3048 "Task cancelled: retention limit exceeded",
3049 true,
3050 );
3051 let bytes = retained_payload_size(&task.task, limits.max_retained_bytes)
3052 .expect("bounded cancellation must fit the aggregate byte limit");
3053 let totals = replacement_totals(
3054 &data,
3055 old_retained,
3056 old_reserved,
3057 bytes,
3058 0,
3059 limits.max_retained_bytes,
3060 )
3061 .expect("bounded cancellation must fit reserved global accounting");
3062 (bytes, totals)
3063 }
3064 };
3065 task.retained_bytes = new_retained;
3066 task.reserved_bytes = 0;
3067 let notify = task.completion_notify.clone();
3068 data.retained_bytes = retained_bytes;
3069 data.reserved_bytes = reserved_bytes;
3070 let object = task.to_task_object();
3071 data.tasks.insert(task_id.to_string(), task);
3072 notify.notify_waiters();
3073 return Ok(Some(object));
3074 }
3075 let object = task.to_task_object();
3076 data.tasks.insert(task_id.to_string(), task);
3077 Ok(Some(object))
3078 }
3079}
3080
3081pub fn tasks_extension() -> crate::ExtensionDeclaration {
3086 crate::ExtensionDeclaration::empty(crate::protocol::TASKS_EXTENSION_ID)
3087 .expect("the built-in Tasks extension declaration is valid")
3088}
3089
3090impl crate::McpRouter {
3091 pub fn with_tasks(self) -> Self {
3111 self.with_protocol_extension(tasks_extension())
3112 }
3113}
3114
3115impl crate::McpClientBuilder {
3116 pub fn with_tasks(self) -> Self {
3118 self.with_protocol_extension(tasks_extension())
3119 }
3120}
3121
3122impl crate::RequestContext {
3123 pub fn supports_tasks(&self) -> bool {
3129 self.negotiated_extensions()
3130 .is_some_and(|extensions| extensions.contains(crate::protocol::TASKS_EXTENSION_ID))
3131 }
3132}
3133
3134fn chrono_now_iso8601() -> String {
3136 use std::time::SystemTime;
3137
3138 let now = SystemTime::now();
3139 let duration = now
3140 .duration_since(SystemTime::UNIX_EPOCH)
3141 .unwrap_or_default();
3142
3143 let secs = duration.as_secs();
3144 let millis = duration.subsec_millis();
3145
3146 let days = secs / 86400;
3149 let remaining = secs % 86400;
3150 let hours = remaining / 3600;
3151 let remaining = remaining % 3600;
3152 let minutes = remaining / 60;
3153 let seconds = remaining % 60;
3154
3155 let mut year = 1970i32;
3158 let mut remaining_days = days as i32;
3159
3160 loop {
3161 let days_in_year = if is_leap_year(year) { 366 } else { 365 };
3162 if remaining_days < days_in_year {
3163 break;
3164 }
3165 remaining_days -= days_in_year;
3166 year += 1;
3167 }
3168
3169 let days_in_months: [i32; 12] = if is_leap_year(year) {
3170 [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
3171 } else {
3172 [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
3173 };
3174
3175 let mut month = 1;
3176 for days_in_month in days_in_months.iter() {
3177 if remaining_days < *days_in_month {
3178 break;
3179 }
3180 remaining_days -= days_in_month;
3181 month += 1;
3182 }
3183
3184 let day = remaining_days + 1;
3185
3186 format!(
3187 "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z",
3188 year, month, day, hours, minutes, seconds, millis
3189 )
3190}
3191
3192fn is_leap_year(year: i32) -> bool {
3193 (year % 4 == 0 && year % 100 != 0) || (year % 400 == 0)
3194}
3195
3196#[cfg(test)]
3197mod tests {
3198 use super::*;
3199 use crate::protocol::{
3200 ElicitAction, ElicitFieldValue, ElicitFormParams, ElicitFormSchema, ElicitRequestParams,
3201 ElicitResult, InputRequest, InputResponse, ListRootsParams,
3202 };
3203
3204 #[test]
3205 fn task_resume_context_constructor_preserves_every_field() {
3206 let input_responses = InputResponses::from([(
3207 "approval".to_string(),
3208 InputResponse::Elicit(ElicitResult {
3209 action: ElicitAction::Accept,
3210 content: None,
3211 meta: None,
3212 }),
3213 )]);
3214 let context = TaskResumeContext::new(
3215 "build_report",
3216 serde_json::json!({"format": "pdf"}),
3217 input_responses.clone(),
3218 );
3219
3220 assert_eq!(context.tool_name, "build_report");
3221 assert_eq!(context.arguments, serde_json::json!({"format": "pdf"}));
3222 assert!(context.cancellation_token.is_none());
3223 assert_eq!(
3224 serde_json::to_value(&context.input_responses).unwrap(),
3225 serde_json::to_value(&input_responses).unwrap()
3226 );
3227 }
3228
3229 #[tokio::test]
3230 async fn test_create_task() {
3231 let store = MemoryTaskStore::new();
3232 let (id, token) = store
3233 .create_task("test-tool", serde_json::json!({"a": 1}), None, None)
3234 .await
3235 .unwrap();
3236
3237 assert!(!id.is_empty());
3238 assert!(!token.is_cancelled());
3239
3240 let info = store
3241 .get_task(&id)
3242 .await
3243 .unwrap()
3244 .expect("task should exist");
3245 assert_eq!(info.task_id, id);
3246 assert_eq!(info.status, TaskStatus::Working);
3247 assert_eq!(info.ttl, Some(300_000));
3248 }
3249
3250 #[tokio::test]
3251 async fn configured_default_ttl_is_used_when_creation_omits_one() {
3252 let store = MemoryTaskStore::with_config(
3253 MemoryTaskStoreConfig::default().default_ttl(Duration::from_secs(42)),
3254 );
3255 let (id, _) = store
3256 .create_task("test-tool", serde_json::json!({}), None, None)
3257 .await
3258 .unwrap();
3259
3260 assert_eq!(
3261 store.get_task(&id).await.unwrap().unwrap().ttl,
3262 Some(42_000)
3263 );
3264 }
3265
3266 #[tokio::test]
3267 async fn working_task_expiry_cancels_and_wakes_completion_waiter() {
3268 let store = MemoryTaskStore::with_config(
3269 MemoryTaskStoreConfig::default()
3270 .default_ttl(Duration::from_secs(60))
3271 .cleanup_interval(Duration::from_secs(60)),
3272 );
3273 let (id, cancellation) = store
3274 .create_task("test-tool", serde_json::json!({}), None, None)
3275 .await
3276 .unwrap();
3277
3278 let waiting_store = store.clone();
3279 let waiting_id = id.clone();
3280 let waiter = tokio::spawn(async move {
3281 waiting_store
3282 .wait_for_completion(&waiting_id)
3283 .await
3284 .unwrap()
3285 });
3286
3287 let mut waiter = waiter;
3290 assert!(
3291 tokio::time::timeout(Duration::from_millis(20), &mut waiter)
3292 .await
3293 .is_err()
3294 );
3295 assert!(store.set_ttl(&id, 0).await.unwrap());
3296
3297 tokio::time::timeout(Duration::from_secs(2), cancellation.cancelled())
3298 .await
3299 .expect("expiry did not cancel the task token");
3300 let snapshot = tokio::time::timeout(Duration::from_secs(2), waiter)
3301 .await
3302 .expect("expiry did not wake the completion waiter")
3303 .unwrap();
3304 assert!(
3305 snapshot.is_none(),
3306 "an expired task has no visible snapshot"
3307 );
3308 assert!(matches!(
3309 store.task_presence(&id).await.unwrap(),
3310 TaskPresence::Expired { .. }
3311 ));
3312 assert!(
3313 !store
3314 .complete_task(&id, CallToolResult::text("late"))
3315 .await
3316 .unwrap(),
3317 "a terminal write must not resurrect an expired task"
3318 );
3319 }
3320
3321 #[tokio::test]
3322 async fn automatic_cleanup_physically_reclaims_expired_tasks() {
3323 let store = MemoryTaskStore::with_config(
3324 MemoryTaskStoreConfig::default()
3325 .default_ttl(Duration::from_secs(60))
3326 .cleanup_interval(Duration::from_millis(25)),
3327 );
3328 let (id, _) = store
3329 .create_task("test-tool", serde_json::json!({}), None, None)
3330 .await
3331 .unwrap();
3332 assert_eq!(store.len(), 1);
3333 assert!(store.set_ttl(&id, 0).await.unwrap());
3334
3335 tokio::time::timeout(Duration::from_secs(2), async {
3336 while !store.is_empty() {
3337 tokio::time::sleep(Duration::from_millis(5)).await;
3338 }
3339 })
3340 .await
3341 .expect("automatic cleanup did not reclaim the expired record");
3342 }
3343
3344 #[tokio::test]
3345 async fn shortening_ttl_reschedules_expiry_from_creation() {
3346 let store = MemoryTaskStore::with_config(
3347 MemoryTaskStoreConfig::default()
3348 .default_ttl(Duration::from_secs(60))
3349 .cleanup_interval(Duration::from_secs(60)),
3350 );
3351 let (id, cancellation) = store
3352 .create_task("test-tool", serde_json::json!({}), None, None)
3353 .await
3354 .unwrap();
3355
3356 assert!(store.set_ttl(&id, 250).await.unwrap());
3357 tokio::time::timeout(Duration::from_secs(3), cancellation.cancelled())
3358 .await
3359 .expect("shorter TTL did not reschedule the expiry wakeup");
3360 assert!(store.get_task(&id).await.unwrap().is_none());
3361 }
3362
3363 #[tokio::test]
3364 async fn manual_and_scheduled_retirement_signal_expiry_only_once() {
3365 let store = MemoryTaskStore::with_config(
3366 MemoryTaskStoreConfig::default()
3367 .default_ttl(Duration::from_secs(60))
3368 .cleanup_interval(Duration::from_secs(60)),
3369 );
3370 let (id, cancellation) = store
3371 .create_task("test-tool", serde_json::json!({}), None, None)
3372 .await
3373 .unwrap();
3374
3375 store
3379 .state
3380 .data
3381 .write()
3382 .unwrap()
3383 .tasks
3384 .get_mut(&id)
3385 .unwrap()
3386 .ttl = 0;
3387 let first = store.state.retire_expired(false);
3388 let second = store.state.retire_expired(false);
3389 assert_eq!(first.signalled + second.signalled, 1);
3390 assert!(cancellation.is_cancelled());
3391 assert_eq!(store.cleanup_expired(), 1);
3392 assert_eq!(store.cleanup_expired(), 0);
3393 }
3394
3395 #[tokio::test]
3396 async fn dropping_the_last_store_clone_releases_worker_state() {
3397 let store = MemoryTaskStore::with_config(
3398 MemoryTaskStoreConfig::default().cleanup_interval(Duration::from_secs(60)),
3399 );
3400 store
3401 .create_task("test-tool", serde_json::json!({}), None, None)
3402 .await
3403 .unwrap();
3404 let weak = Arc::downgrade(&store.state);
3405 drop(store);
3406 tokio::time::timeout(Duration::from_secs(2), async {
3407 while weak.upgrade().is_some() {
3408 tokio::task::yield_now().await;
3409 }
3410 })
3411 .await
3412 .expect("the expiry worker retained the store's task state");
3413 }
3414
3415 #[test]
3416 fn creating_a_task_does_not_require_a_tokio_runtime() {
3417 let store = MemoryTaskStore::with_config(
3418 MemoryTaskStoreConfig::default()
3419 .default_ttl(Duration::from_secs(60))
3420 .cleanup_interval(Duration::from_secs(60)),
3421 );
3422 let (id, token) = futures::executor::block_on(store.create_task(
3423 "test-tool",
3424 serde_json::json!({}),
3425 None,
3426 None,
3427 ))
3428 .expect("the standard-thread worker should start without Tokio");
3429
3430 assert!(!token.is_cancelled());
3431 assert!(
3432 futures::executor::block_on(store.get_task(&id))
3433 .unwrap()
3434 .is_some()
3435 );
3436 }
3437
3438 #[tokio::test]
3439 async fn test_task_lifecycle() {
3440 let store = MemoryTaskStore::new();
3441 let (id, _) = store
3442 .create_task("test-tool", serde_json::json!({}), None, None)
3443 .await
3444 .unwrap();
3445
3446 assert!(
3448 store
3449 .complete_task(&id, CallToolResult::text("Done"))
3450 .await
3451 .unwrap()
3452 );
3453
3454 let info = store.get_task(&id).await.unwrap().unwrap();
3455 assert_eq!(info.status, TaskStatus::Completed);
3456 }
3457
3458 #[tokio::test]
3459 async fn test_task_cancellation() {
3460 let store = MemoryTaskStore::new();
3461 let (id, token) = store
3462 .create_task("test-tool", serde_json::json!({}), None, None)
3463 .await
3464 .unwrap();
3465
3466 assert!(!token.is_cancelled());
3467
3468 let task_obj = store
3469 .cancel_task(&id, Some("User requested"))
3470 .await
3471 .unwrap();
3472 assert!(task_obj.is_some());
3473 assert_eq!(task_obj.unwrap().status, TaskStatus::Cancelled);
3474 assert!(token.is_cancelled());
3475
3476 let info = store.get_task(&id).await.unwrap().unwrap();
3477 assert_eq!(info.status, TaskStatus::Cancelled);
3478 }
3479
3480 #[tokio::test]
3481 async fn test_task_failure() {
3482 let store = MemoryTaskStore::new();
3483 let (id, _) = store
3484 .create_task("test-tool", serde_json::json!({}), None, None)
3485 .await
3486 .unwrap();
3487
3488 assert!(
3489 store
3490 .fail_task(&id, JsonRpcError::internal_error("Something went wrong"))
3491 .await
3492 .unwrap()
3493 );
3494
3495 let info = store.get_task(&id).await.unwrap().unwrap();
3496 assert_eq!(info.status, TaskStatus::Failed);
3497 assert_eq!(
3498 info.status_message.as_deref(),
3499 Some("Task failed: Something went wrong")
3500 );
3501 }
3502
3503 #[tokio::test]
3504 async fn test_list_tasks() {
3505 let store = MemoryTaskStore::new();
3506 store
3507 .create_task("tool1", serde_json::json!({}), None, None)
3508 .await
3509 .unwrap();
3510 store
3511 .create_task("tool2", serde_json::json!({}), None, None)
3512 .await
3513 .unwrap();
3514 let (id3, _) = store
3515 .create_task("tool3", serde_json::json!({}), None, None)
3516 .await
3517 .unwrap();
3518
3519 store
3521 .complete_task(&id3, CallToolResult::text("Done"))
3522 .await
3523 .unwrap();
3524
3525 let all = store.list_tasks(None).await.unwrap();
3527 assert_eq!(all.len(), 3);
3528
3529 let working = store.list_tasks(Some(TaskStatus::Working)).await.unwrap();
3531 assert_eq!(working.len(), 2);
3532
3533 let completed = store.list_tasks(Some(TaskStatus::Completed)).await.unwrap();
3535 assert_eq!(completed.len(), 1);
3536 }
3537
3538 #[tokio::test]
3539 async fn test_terminal_state_immutable() {
3540 let store = MemoryTaskStore::new();
3541 let (id, _) = store
3542 .create_task("test-tool", serde_json::json!({}), None, None)
3543 .await
3544 .unwrap();
3545
3546 store
3548 .complete_task(&id, CallToolResult::text("Done"))
3549 .await
3550 .unwrap();
3551
3552 assert!(
3554 !store
3555 .fail_task(&id, JsonRpcError::internal_error("Error"))
3556 .await
3557 .unwrap()
3558 );
3559
3560 let info = store.get_task(&id).await.unwrap().unwrap();
3562 assert_eq!(info.status, TaskStatus::Completed);
3563 }
3564
3565 #[tokio::test]
3566 async fn test_task_ids_unique() {
3567 let store = MemoryTaskStore::new();
3568 let (id1, _) = store
3569 .create_task("tool", serde_json::json!({}), None, None)
3570 .await
3571 .unwrap();
3572 let (id2, _) = store
3573 .create_task("tool", serde_json::json!({}), None, None)
3574 .await
3575 .unwrap();
3576 let (id3, _) = store
3577 .create_task("tool", serde_json::json!({}), None, None)
3578 .await
3579 .unwrap();
3580
3581 assert_ne!(id1, id2);
3582 assert_ne!(id2, id3);
3583 assert_ne!(id1, id3);
3584 }
3585
3586 #[tokio::test]
3587 async fn test_get_task_result() {
3588 let store = MemoryTaskStore::new();
3589 let (id, _) = store
3590 .create_task("test-tool", serde_json::json!({}), None, None)
3591 .await
3592 .unwrap();
3593
3594 let result = CallToolResult::text("The result");
3596 store.complete_task(&id, result).await.unwrap();
3597
3598 let (task_obj, result, error) = store.get_task_result(&id).await.unwrap().unwrap();
3599 assert_eq!(task_obj.status, TaskStatus::Completed);
3600 assert!(result.is_some());
3601 assert!(error.is_none());
3602 }
3603
3604 #[tokio::test]
3605 async fn test_wait_for_completion_returns_terminal_snapshot() {
3606 let store = MemoryTaskStore::new();
3607 let (id, _) = store
3608 .create_task("test-tool", serde_json::json!({}), None, None)
3609 .await
3610 .unwrap();
3611
3612 let waiter_store = store.clone();
3614 let waiter_id = id.clone();
3615 let waiter =
3616 tokio::spawn(async move { waiter_store.wait_for_completion(&waiter_id).await });
3617
3618 tokio::time::sleep(Duration::from_millis(10)).await;
3619 store
3620 .complete_task(&id, CallToolResult::text("Done"))
3621 .await
3622 .unwrap();
3623
3624 let (task_obj, result, error) = waiter.await.unwrap().unwrap().unwrap();
3625 assert_eq!(task_obj.status, TaskStatus::Completed);
3626 assert!(result.is_some());
3627 assert!(error.is_none());
3628 }
3629
3630 #[tokio::test]
3631 async fn wait_for_completion_does_not_treat_live_cancellation_signal_as_terminal() {
3632 let store = MemoryTaskStore::new();
3633 let (id, cancellation) = store
3634 .create_task("test-tool", serde_json::json!({}), None, None)
3635 .await
3636 .unwrap();
3637 let waiter_store = store.clone();
3638 let waiter_id = id.clone();
3639 let mut waiter =
3640 tokio::spawn(
3641 async move { waiter_store.wait_for_completion(&waiter_id).await.unwrap() },
3642 );
3643
3644 cancellation.cancel();
3645 assert!(
3646 tokio::time::timeout(Duration::from_millis(20), &mut waiter)
3647 .await
3648 .is_err(),
3649 "the cooperative signal alone must not return a working snapshot"
3650 );
3651
3652 store
3653 .cancel_task(&id, Some("teardown complete"))
3654 .await
3655 .unwrap();
3656 let (task, _, _) = waiter.await.unwrap().unwrap();
3657 assert_eq!(task.status, TaskStatus::Cancelled);
3658 }
3659
3660 #[tokio::test]
3661 async fn dyn_task_store_object_safe() {
3662 let store: Arc<dyn TaskStore> = Arc::new(MemoryTaskStore::new());
3664 let (id, _) = store
3665 .create_task("tool", serde_json::json!({}), None, None)
3666 .await
3667 .unwrap();
3668 assert!(store.get_task(&id).await.unwrap().is_some());
3669 }
3670
3671 #[test]
3672 fn test_iso8601_timestamp() {
3673 let ts = chrono_now_iso8601();
3674 assert!(ts.ends_with('Z'));
3676 assert!(ts.contains('T'));
3677 assert_eq!(ts.len(), 24); }
3679
3680 #[test]
3681 fn test_task_status_display() {
3682 assert_eq!(TaskStatus::Working.to_string(), "working");
3683 assert_eq!(TaskStatus::InputRequired.to_string(), "input_required");
3684 assert_eq!(TaskStatus::Completed.to_string(), "completed");
3685 assert_eq!(TaskStatus::Failed.to_string(), "failed");
3686 assert_eq!(TaskStatus::Cancelled.to_string(), "cancelled");
3687 }
3688
3689 #[test]
3690 fn test_task_status_is_terminal() {
3691 assert!(!TaskStatus::Working.is_terminal());
3692 assert!(!TaskStatus::InputRequired.is_terminal());
3693 assert!(TaskStatus::Completed.is_terminal());
3694 assert!(TaskStatus::Failed.is_terminal());
3695 assert!(TaskStatus::Cancelled.is_terminal());
3696 }
3697
3698 fn requests(keys: &[&str]) -> InputRequests {
3699 keys.iter()
3700 .map(|k| {
3701 (
3702 k.to_string(),
3703 InputRequest::ListRoots(ListRootsParams { meta: None }),
3704 )
3705 })
3706 .collect()
3707 }
3708
3709 fn accept(key: &str) -> (String, InputResponse) {
3710 (
3711 key.to_string(),
3712 InputResponse::Elicit(ElicitResult {
3713 action: ElicitAction::Accept,
3714 content: None,
3715 meta: None,
3716 }),
3717 )
3718 }
3719
3720 async fn working_task(store: &MemoryTaskStore, ttl: Option<u64>) -> String {
3721 store
3722 .create_task("tool", serde_json::json!({}), ttl, None)
3723 .await
3724 .unwrap()
3725 .0
3726 }
3727
3728 #[tokio::test]
3729 async fn task_ids_are_unguessable_not_sequential() {
3730 let store = MemoryTaskStore::new();
3731 let mut ids = BTreeSet::new();
3732 for _ in 0..64 {
3733 ids.insert(working_task(&store, None).await);
3734 }
3735 assert_eq!(ids.len(), 64, "task IDs collided");
3736
3737 for id in &ids {
3738 assert_eq!(id.len(), 32, "expected 128 bits of hex: {id}");
3739 assert!(id.chars().all(|c| c.is_ascii_hexdigit()), "{id}");
3740 assert!(!id.starts_with("task-"), "sequential-looking ID: {id}");
3741 }
3742
3743 let leading: BTreeSet<&str> = ids.iter().map(|id| &id[..2]).collect();
3746 assert!(
3747 leading.len() > 32,
3748 "only {} distinct leading bytes across 64 IDs",
3749 leading.len()
3750 );
3751 }
3752
3753 #[tokio::test]
3754 async fn ttl_runs_from_creation_and_expired_tasks_read_as_absent() {
3755 let store = MemoryTaskStore::new();
3756 let id = working_task(&store, Some(0)).await;
3757
3758 tokio::time::sleep(Duration::from_millis(5)).await;
3761
3762 assert!(store.get_task(&id).await.unwrap().is_none());
3763 assert!(store.get_task_result(&id).await.unwrap().is_none());
3764 assert!(store.list_tasks(None).await.unwrap().is_empty());
3765 assert!(
3766 store
3767 .outstanding_input_requests(&id)
3768 .await
3769 .unwrap()
3770 .is_none()
3771 );
3772 assert!(store.cancel_task(&id, None).await.unwrap().is_none());
3773 assert!(!store.set_ttl(&id, 60_000).await.unwrap());
3774 assert!(
3775 !store
3776 .complete_task(&id, CallToolResult::text("late"))
3777 .await
3778 .unwrap()
3779 );
3780 }
3781
3782 #[tokio::test]
3783 async fn ttl_is_mutable_over_the_task_lifetime() {
3784 let store = MemoryTaskStore::new();
3785 let id = working_task(&store, Some(60_000)).await;
3786
3787 assert!(store.set_ttl(&id, 120_000).await.unwrap());
3788 let task = store.get_task(&id).await.unwrap().unwrap();
3789 assert_eq!(task.ttl, Some(120_000));
3790
3791 assert!(store.set_ttl(&id, 0).await.unwrap());
3793 tokio::time::sleep(Duration::from_millis(5)).await;
3794 assert!(store.get_task(&id).await.unwrap().is_none());
3795 }
3796
3797 #[tokio::test]
3798 async fn require_input_records_requests_and_exposes_them() {
3799 let store = MemoryTaskStore::new();
3800 let id = working_task(&store, None).await;
3801
3802 assert!(
3803 store
3804 .require_input(&id, requests(&["approval", "region"]), Some("need input"))
3805 .await
3806 .unwrap()
3807 );
3808
3809 let task = store.get_task(&id).await.unwrap().unwrap();
3810 assert_eq!(task.status, TaskStatus::InputRequired);
3811 assert_eq!(task.status_message.as_deref(), Some("need input"));
3812
3813 let outstanding = store
3814 .outstanding_input_requests(&id)
3815 .await
3816 .unwrap()
3817 .unwrap();
3818 assert_eq!(
3819 outstanding.keys().collect::<Vec<_>>(),
3820 vec!["approval", "region"],
3821 "every outstanding request must be exposed, not just the newest"
3822 );
3823 }
3824
3825 #[tokio::test]
3826 async fn partial_input_responses_leave_the_rest_outstanding() {
3827 let store = MemoryTaskStore::new();
3828 let id = working_task(&store, None).await;
3829 store
3830 .require_input(&id, requests(&["approval", "region"]), None)
3831 .await
3832 .unwrap();
3833
3834 let applied = store
3835 .apply_input_responses(&id, [accept("approval")].into_iter().collect())
3836 .await
3837 .unwrap()
3838 .unwrap();
3839
3840 assert_eq!(applied.accepted, ["approval".to_string()].into());
3841 assert!(applied.ignored.is_empty());
3842 assert_eq!(applied.still_outstanding, ["region".to_string()].into());
3843 assert!(!applied.is_complete());
3844
3845 let task = store.get_task(&id).await.unwrap().unwrap();
3847 assert_eq!(task.status, TaskStatus::InputRequired);
3848 assert_eq!(
3849 store
3850 .outstanding_input_requests(&id)
3851 .await
3852 .unwrap()
3853 .unwrap()
3854 .keys()
3855 .collect::<Vec<_>>(),
3856 vec!["region"]
3857 );
3858
3859 let applied = store
3861 .apply_input_responses(&id, [accept("region")].into_iter().collect())
3862 .await
3863 .unwrap()
3864 .unwrap();
3865 assert!(applied.is_complete());
3866 assert_eq!(
3867 store.get_task(&id).await.unwrap().unwrap().status,
3868 TaskStatus::Working
3869 );
3870 }
3871
3872 #[tokio::test]
3873 async fn unknown_answered_and_superseded_response_keys_are_ignored() {
3874 let store = MemoryTaskStore::new();
3875 let id = working_task(&store, None).await;
3876 store
3877 .require_input(&id, requests(&["approval", "stale"]), None)
3878 .await
3879 .unwrap();
3880
3881 store
3883 .apply_input_responses(&id, [accept("approval")].into_iter().collect())
3884 .await
3885 .unwrap()
3886 .unwrap();
3887 store
3888 .require_input(&id, requests(&["region"]), None)
3889 .await
3890 .unwrap();
3891
3892 let applied = store
3893 .apply_input_responses(
3894 &id,
3895 [accept("never-issued"), accept("approval"), accept("stale")]
3896 .into_iter()
3897 .collect(),
3898 )
3899 .await
3900 .unwrap()
3901 .unwrap();
3902
3903 assert!(
3904 applied.accepted.is_empty(),
3905 "none of these keys are outstanding"
3906 );
3907 assert_eq!(
3908 applied.ignored,
3909 [
3910 "never-issued".to_string(),
3911 "approval".to_string(),
3912 "stale".to_string()
3913 ]
3914 .into(),
3915 "unknown, already-answered, and superseded keys are all ignored"
3916 );
3917 assert_eq!(applied.still_outstanding, ["region".to_string()].into());
3918 assert_eq!(
3919 store.get_task(&id).await.unwrap().unwrap().status,
3920 TaskStatus::InputRequired,
3921 "ignoring a stale update must not resume or fail the task"
3922 );
3923 }
3924
3925 #[tokio::test]
3930 async fn an_answered_key_cannot_be_reissued() {
3931 let store = MemoryTaskStore::new();
3932 let id = working_task(&store, None).await;
3933 store
3934 .require_input(&id, requests(&["approval"]), None)
3935 .await
3936 .unwrap();
3937 store
3938 .apply_input_responses(&id, [accept("approval")].into_iter().collect())
3939 .await
3940 .unwrap()
3941 .unwrap();
3942
3943 let error = store
3944 .require_input(&id, requests(&["approval"]), None)
3945 .await
3946 .expect_err("an answered key must not be reissued");
3947 assert!(
3948 error.to_string().contains("approval"),
3949 "the message must name the offending key: {error}"
3950 );
3951 }
3952
3953 #[tokio::test]
3957 async fn an_outstanding_request_can_be_carried_forward() {
3958 let store = MemoryTaskStore::new();
3959 let id = working_task(&store, None).await;
3960 store
3961 .require_input(&id, requests(&["approval"]), None)
3962 .await
3963 .unwrap();
3964
3965 assert!(
3967 store
3968 .require_input(&id, requests(&["approval", "region"]), None)
3969 .await
3970 .unwrap()
3971 );
3972
3973 let outstanding = store
3974 .outstanding_input_requests(&id)
3975 .await
3976 .unwrap()
3977 .unwrap();
3978 assert!(outstanding.contains_key("approval"));
3979 assert!(outstanding.contains_key("region"));
3980
3981 let applied = store
3983 .apply_input_responses(
3984 &id,
3985 [accept("approval"), accept("region")].into_iter().collect(),
3986 )
3987 .await
3988 .unwrap()
3989 .unwrap();
3990 assert_eq!(
3991 applied.accepted,
3992 ["approval".to_string(), "region".to_string()].into()
3993 );
3994 assert!(applied.is_complete());
3995 }
3996
3997 #[tokio::test]
4001 async fn an_outstanding_key_cannot_change_what_it_asks() {
4002 use crate::protocol::{ElicitFormParams, ElicitFormSchema, ElicitRequestParams};
4003
4004 let store = MemoryTaskStore::new();
4005 let id = working_task(&store, None).await;
4006 store
4007 .require_input(&id, requests(&["approval"]), None)
4008 .await
4009 .unwrap();
4010
4011 let mut changed: InputRequests = Default::default();
4012 changed.insert(
4013 "approval".to_string(),
4014 InputRequest::Elicit(ElicitRequestParams::Form(ElicitFormParams {
4015 mode: None,
4016 message: "a different question".to_string(),
4017 requested_schema: ElicitFormSchema::new(),
4018 meta: None,
4019 })),
4020 );
4021 store
4022 .require_input(&id, changed, None)
4023 .await
4024 .expect_err("a live key must not be repointed at another request");
4025 }
4026
4027 #[tokio::test]
4031 async fn a_superseded_key_cannot_be_reissued() {
4032 let store = MemoryTaskStore::new();
4033 let id = working_task(&store, None).await;
4034 store
4035 .require_input(&id, requests(&["approval"]), None)
4036 .await
4037 .unwrap();
4038 store
4040 .require_input(&id, requests(&["region"]), None)
4041 .await
4042 .unwrap();
4043
4044 store
4045 .require_input(&id, requests(&["approval"]), None)
4046 .await
4047 .expect_err("a superseded key must not be reissued");
4048 }
4049
4050 #[tokio::test]
4053 async fn distinct_keys_across_rounds_are_fine() {
4054 let store = MemoryTaskStore::new();
4055 let id = working_task(&store, None).await;
4056 store
4057 .require_input(&id, requests(&["approval"]), None)
4058 .await
4059 .unwrap();
4060 store
4061 .apply_input_responses(&id, [accept("approval")].into_iter().collect())
4062 .await
4063 .unwrap()
4064 .unwrap();
4065
4066 assert!(
4067 store
4068 .require_input(&id, requests(&["approval_2"]), None)
4069 .await
4070 .unwrap()
4071 );
4072 let applied = store
4073 .apply_input_responses(&id, [accept("approval_2")].into_iter().collect())
4074 .await
4075 .unwrap()
4076 .unwrap();
4077 assert_eq!(applied.accepted, ["approval_2".to_string()].into());
4078 assert!(applied.is_complete());
4079 }
4080
4081 #[tokio::test]
4082 async fn failed_tasks_preserve_the_structured_error() {
4083 let store = MemoryTaskStore::new();
4084 let id = working_task(&store, None).await;
4085
4086 let mut error = JsonRpcError::invalid_params("bad region");
4087 error.data = Some(serde_json::json!({"field": "region"}));
4088 assert!(store.fail_task(&id, error).await.unwrap());
4089
4090 let (_, result, error) = store.get_task_result(&id).await.unwrap().unwrap();
4091 assert!(result.is_none());
4092 let error = error.expect("structured error must survive the store");
4093 assert_eq!(
4094 error.code, -32602,
4095 "the original code must not be flattened"
4096 );
4097 assert_eq!(error.message, "bad region");
4098 assert_eq!(error.data.unwrap()["field"], "region");
4099 }
4100
4101 #[tokio::test]
4102 async fn tool_error_results_complete_the_task() {
4103 let store = MemoryTaskStore::new();
4104 let id = working_task(&store, None).await;
4105
4106 let mut result = CallToolResult::text("domain failure");
4107 result.is_error = true;
4108 assert!(store.complete_task(&id, result).await.unwrap());
4109
4110 let (task, result, error) = store.get_task_result(&id).await.unwrap().unwrap();
4111 assert_eq!(
4112 task.status,
4113 TaskStatus::Completed,
4114 "isError is a domain error, not an execution failure"
4115 );
4116 assert!(result.unwrap().is_error);
4117 assert!(error.is_none(), "no JSON-RPC error accompanies isError");
4118 }
4119
4120 #[tokio::test]
4121 async fn tasks_record_their_creating_principal() {
4122 let store = MemoryTaskStore::new();
4123 let (owned, _) = store
4124 .create_task("tool", serde_json::json!({}), None, Some("alice".into()))
4125 .await
4126 .unwrap();
4127 let (unowned, _) = store
4128 .create_task("tool", serde_json::json!({}), None, None)
4129 .await
4130 .unwrap();
4131
4132 assert_eq!(
4133 store.task_owner(&owned).await.unwrap(),
4134 Some(Some("alice".to_string()))
4135 );
4136 assert_eq!(store.task_owner(&unowned).await.unwrap(), Some(None));
4137 assert_eq!(
4138 store.task_owner("does-not-exist").await.unwrap(),
4139 None,
4140 "an unknown task has no owner record at all"
4141 );
4142
4143 let wire = serde_json::to_value(store.get_task(&owned).await.unwrap().unwrap()).unwrap();
4145 assert!(
4146 wire.get("owner").is_none(),
4147 "owner leaked to the wire: {wire}"
4148 );
4149 assert!(!wire.to_string().contains("alice"));
4150 }
4151
4152 #[test]
4153 fn owner_matching_is_equality_not_leniency() {
4154 assert!(owner_matches(&None, None), "no auth configured");
4155 assert!(owner_matches(&Some("alice".into()), Some("alice")));
4156
4157 assert!(
4158 !owner_matches(&Some("alice".into()), Some("bob")),
4159 "a different principal must not inherit the task"
4160 );
4161 assert!(
4162 !owner_matches(&Some("alice".into()), None),
4163 "dropping the token must not grant access"
4164 );
4165 assert!(
4166 !owner_matches(&None, Some("alice")),
4167 "an unowned task belongs to a different security context"
4168 );
4169 }
4170
4171 #[tokio::test]
4172 async fn terminal_states_clear_outstanding_requests() {
4173 for (label, terminate) in [("completed", true), ("cancelled", false)] {
4174 let store = MemoryTaskStore::new();
4175 let id = working_task(&store, None).await;
4176 store
4177 .require_input(&id, requests(&["approval"]), None)
4178 .await
4179 .unwrap();
4180
4181 if terminate {
4182 store
4183 .complete_task(&id, CallToolResult::text("done"))
4184 .await
4185 .unwrap();
4186 } else {
4187 store.cancel_task(&id, None).await.unwrap();
4188 }
4189
4190 assert!(
4191 store
4192 .outstanding_input_requests(&id)
4193 .await
4194 .unwrap()
4195 .unwrap()
4196 .is_empty(),
4197 "{label} task still advertises outstanding input requests"
4198 );
4199 assert!(
4200 store
4201 .apply_input_responses(&id, [accept("approval")].into_iter().collect())
4202 .await
4203 .unwrap()
4204 .is_none(),
4205 "{label} task accepted a late input response"
4206 );
4207 }
4208 }
4209
4210 fn store_with_limits(limits: TaskRetentionLimits) -> MemoryTaskStore {
4211 MemoryTaskStore::with_retention_limits(limits)
4212 }
4213
4214 fn assert_accounting(store: &MemoryTaskStore) {
4215 let data = store.state.data.read().unwrap();
4216 let retained_bytes = data
4217 .tasks
4218 .values()
4219 .map(|task| {
4220 let measured = retained_payload_size(&task.task, usize::MAX).unwrap();
4221 assert_eq!(task.retained_bytes, measured);
4222 measured
4223 })
4224 .sum::<usize>();
4225 let reserved_bytes = data
4226 .tasks
4227 .values()
4228 .map(|task| task.reserved_bytes)
4229 .sum::<usize>();
4230 assert_eq!(data.retained_bytes, retained_bytes);
4231 assert_eq!(data.reserved_bytes, reserved_bytes);
4232 let expected = data.usage();
4233 drop(data);
4234 assert_eq!(store.usage(), expected);
4235 assert!(store.usage().charged_bytes() <= store.state.retention_limits.max_retained_bytes);
4236 }
4237
4238 fn large_request(key: &str, message: String) -> InputRequests {
4239 [(
4240 key.to_string(),
4241 InputRequest::Elicit(ElicitRequestParams::Form(ElicitFormParams {
4242 mode: None,
4243 message,
4244 requested_schema: ElicitFormSchema::new(),
4245 meta: None,
4246 })),
4247 )]
4248 .into_iter()
4249 .collect()
4250 }
4251
4252 fn accept_text(key: &str, value: String) -> (String, InputResponse) {
4253 (
4254 key.to_string(),
4255 InputResponse::Elicit(ElicitResult::accept(std::collections::HashMap::from([(
4256 "value".to_string(),
4257 ElicitFieldValue::String(value),
4258 )]))),
4259 )
4260 }
4261
4262 #[test]
4263 fn retention_policy_defaults_are_finite_and_unbounded_is_explicit() {
4264 assert_eq!(TaskRetentionLimits::new(), TaskRetentionLimits::default());
4265 let limits = TaskRetentionLimits::default();
4266 assert_eq!(limits.max_tasks, 1_024);
4267 assert_eq!(limits.max_payload_bytes, 4 * 1024 * 1024);
4268 assert_eq!(limits.max_retained_bytes, 64 * 1024 * 1024);
4269
4270 let unbounded = TaskRetentionLimits::unbounded();
4271 assert_eq!(unbounded.max_tasks, usize::MAX);
4272 assert_eq!(unbounded.max_payload_bytes, usize::MAX);
4273 assert_eq!(unbounded.max_retained_bytes, usize::MAX);
4274
4275 let synthetic_overflow = TaskStoreUsage {
4276 retained_bytes: usize::MAX,
4277 reserved_bytes: 1,
4278 ..TaskStoreUsage::default()
4279 };
4280 assert_eq!(synthetic_overflow.charged_bytes(), usize::MAX);
4281
4282 let config = MemoryTaskStoreConfig {
4284 default_ttl: Duration::from_secs(30),
4285 cleanup_interval: Duration::from_secs(10),
4286 };
4287 let default_limited = MemoryTaskStore::with_config(config);
4288 assert_eq!(
4289 default_limited.state.retention_limits,
4290 TaskRetentionLimits::default()
4291 );
4292 let explicitly_unbounded =
4293 MemoryTaskStore::with_config_and_retention(config, TaskRetentionLimits::unbounded());
4294 assert_eq!(
4295 explicitly_unbounded.state.retention_limits,
4296 TaskRetentionLimits::unbounded()
4297 );
4298 }
4299
4300 #[test]
4301 fn payload_validation_uses_one_stricter_counting_pass() {
4302 struct Counted<'a>(&'a std::sync::atomic::AtomicUsize);
4303
4304 impl serde::Serialize for Counted<'_> {
4305 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
4306 where
4307 S: serde::Serializer,
4308 {
4309 self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
4310 serializer.serialize_str("payload")
4311 }
4312 }
4313
4314 let calls = std::sync::atomic::AtomicUsize::new(0);
4315 let limits = TaskRetentionLimits::unbounded().max_retained_bytes(64);
4316 assert_eq!(validate_payload(&Counted(&calls), limits).unwrap(), 9);
4317 assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 1);
4318
4319 let tied = TaskRetentionLimits::unbounded()
4320 .max_payload_bytes(1)
4321 .max_retained_bytes(1);
4322 assert!(matches!(
4323 validate_payload("x", tied),
4324 Err(TaskStoreError::RetentionLimitExceeded {
4325 kind: TaskRetentionLimitKind::PayloadBytes,
4326 limit: 1
4327 })
4328 ));
4329 let aggregate_is_stricter = TaskRetentionLimits::unbounded().max_retained_bytes(4);
4330 assert!(matches!(
4331 validate_prefixed_string("x", "prefix: ", aggregate_is_stricter),
4332 Err(TaskStoreError::RetentionLimitExceeded {
4333 kind: TaskRetentionLimitKind::AggregateBytes,
4334 limit: 4
4335 })
4336 ));
4337 }
4338
4339 #[tokio::test]
4340 async fn payload_limit_accepts_exact_boundary_and_rejects_without_mutation() {
4341 let arguments = serde_json::json!({"data": "abcd"});
4342 let exact = serde_json::to_vec(&arguments).unwrap().len();
4343 let store = store_with_limits(TaskRetentionLimits::unbounded().max_payload_bytes(exact));
4344 store
4345 .create_task("tool", arguments.clone(), None, None)
4346 .await
4347 .expect("the exact encoded-byte boundary is inclusive");
4348 let before = store.usage();
4349
4350 let error = store
4351 .create_task("tool", serde_json::json!({"data": "abcde"}), None, None)
4352 .await
4353 .unwrap_err();
4354 assert!(matches!(
4355 error,
4356 TaskStoreError::RetentionLimitExceeded {
4357 kind: TaskRetentionLimitKind::PayloadBytes,
4358 limit
4359 } if limit == exact
4360 ));
4361 assert_eq!(store.usage(), before);
4362 assert_accounting(&store);
4363
4364 let zero = store_with_limits(TaskRetentionLimits::unbounded().max_payload_bytes(0));
4365 assert!(matches!(
4366 zero.create_task("tool", serde_json::Value::Null, None, None)
4367 .await,
4368 Err(TaskStoreError::RetentionLimitExceeded {
4369 kind: TaskRetentionLimitKind::PayloadBytes,
4370 limit: 0
4371 })
4372 ));
4373 }
4374
4375 #[tokio::test]
4376 async fn oversized_owner_rejects_create_atomically() {
4377 let store = store_with_limits(
4378 TaskRetentionLimits::unbounded()
4379 .max_payload_bytes(64)
4380 .max_retained_bytes(4 * 1024),
4381 );
4382 let error = store
4383 .create_task(
4384 "tool",
4385 serde_json::Value::Null,
4386 None,
4387 Some("owner-secret".repeat(128)),
4388 )
4389 .await
4390 .unwrap_err();
4391 assert!(matches!(
4392 error,
4393 TaskStoreError::RetentionLimitExceeded {
4394 kind: TaskRetentionLimitKind::PayloadBytes,
4395 limit: 64
4396 }
4397 ));
4398 assert_eq!(store.usage(), TaskStoreUsage::default());
4399 assert_accounting(&store);
4400 }
4401
4402 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
4403 async fn concurrent_task_count_admission_never_overshoots() {
4404 let store = store_with_limits(TaskRetentionLimits::unbounded().max_tasks(1));
4405 let barrier = Arc::new(tokio::sync::Barrier::new(17));
4406 let mut creates = Vec::new();
4407 for _ in 0..16 {
4408 let store = store.clone();
4409 let barrier = barrier.clone();
4410 creates.push(tokio::spawn(async move {
4411 barrier.wait().await;
4412 store
4413 .create_task("tool", serde_json::json!({}), None, None)
4414 .await
4415 }));
4416 }
4417 barrier.wait().await;
4418
4419 let mut accepted = 0;
4420 for create in creates {
4421 match create.await.unwrap() {
4422 Ok(_) => accepted += 1,
4423 Err(TaskStoreError::RetentionLimitExceeded {
4424 kind: TaskRetentionLimitKind::TaskCount,
4425 limit: 1,
4426 }) => {}
4427 Err(other) => panic!("unexpected create error: {other}"),
4428 }
4429 }
4430 assert_eq!(accepted, 1);
4431 assert_eq!(store.usage().task_count, 1);
4432 assert_accounting(&store);
4433 }
4434
4435 #[tokio::test]
4436 async fn replacement_and_input_accumulation_account_exactly() {
4437 let store = store_with_limits(TaskRetentionLimits::unbounded());
4438 let id = working_task(&store, None).await;
4439 let initial = store.usage();
4440
4441 assert!(
4442 store
4443 .set_task_meta(&id, serde_json::json!({"note": "x".repeat(2_000)}))
4444 .await
4445 .unwrap()
4446 );
4447 let large_meta = store.usage();
4448 assert!(large_meta.retained_bytes > initial.retained_bytes);
4449 assert_accounting(&store);
4450
4451 assert!(
4452 store
4453 .set_task_meta(&id, serde_json::json!({"note": "small"}))
4454 .await
4455 .unwrap()
4456 );
4457 let small_meta = store.usage();
4458 assert!(small_meta.retained_bytes < large_meta.retained_bytes);
4459
4460 store
4461 .require_input(&id, large_request("first", "question".repeat(20)), None)
4462 .await
4463 .unwrap();
4464 let requested = store.usage();
4465 assert!(requested.retained_bytes > small_meta.retained_bytes);
4466 store
4467 .apply_input_responses(
4468 &id,
4469 [accept_text("first", "answer".repeat(30))]
4470 .into_iter()
4471 .collect(),
4472 )
4473 .await
4474 .unwrap()
4475 .unwrap();
4476 let first_answer = store.usage();
4477 assert_accounting(&store);
4478
4479 store
4480 .require_input(&id, large_request("second", "next".repeat(20)), None)
4481 .await
4482 .unwrap();
4483 store
4484 .apply_input_responses(
4485 &id,
4486 [accept_text("second", "more".repeat(40))]
4487 .into_iter()
4488 .collect(),
4489 )
4490 .await
4491 .unwrap()
4492 .unwrap();
4493 assert!(store.usage().retained_bytes > first_answer.retained_bytes);
4494 assert_accounting(&store);
4495 }
4496
4497 #[tokio::test]
4498 async fn rejected_payload_mutations_are_atomic_and_ignored_input_is_not_charged() {
4499 let limits = TaskRetentionLimits::unbounded().max_payload_bytes(256);
4500 let store = store_with_limits(limits);
4501 let id = working_task(&store, None).await;
4502
4503 let before_meta = store.usage();
4504 assert!(matches!(
4505 store
4506 .set_task_meta(&id, serde_json::json!({"secret": "m".repeat(1_024)}))
4507 .await,
4508 Err(TaskStoreError::RetentionLimitExceeded {
4509 kind: TaskRetentionLimitKind::PayloadBytes,
4510 ..
4511 })
4512 ));
4513 assert_eq!(store.usage(), before_meta);
4514
4515 assert!(matches!(
4516 store
4517 .require_input(&id, large_request("large", "q".repeat(1_024)), None)
4518 .await,
4519 Err(TaskStoreError::RetentionLimitExceeded {
4520 kind: TaskRetentionLimitKind::PayloadBytes,
4521 ..
4522 })
4523 ));
4524 assert_eq!(
4525 store.get_task(&id).await.unwrap().unwrap().status,
4526 TaskStatus::Working
4527 );
4528
4529 store
4530 .require_input(&id, requests(&["approval"]), None)
4531 .await
4532 .unwrap();
4533 let accepted = store
4534 .apply_input_responses(
4535 &id,
4536 [
4537 accept("approval"),
4538 accept_text("ignored", "never-retained".repeat(1_024)),
4539 ]
4540 .into_iter()
4541 .collect(),
4542 )
4543 .await
4544 .unwrap()
4545 .unwrap();
4546 assert_eq!(accepted.accepted, ["approval".to_string()].into());
4547 assert_eq!(accepted.ignored, ["ignored".to_string()].into());
4548 assert_accounting(&store);
4549
4550 let other = working_task(&store, None).await;
4551 store
4552 .require_input(&other, requests(&["approval"]), None)
4553 .await
4554 .unwrap();
4555 let before_response = store.usage();
4556 assert!(matches!(
4557 store
4558 .apply_input_responses(
4559 &other,
4560 [accept_text("approval", "secret".repeat(1_024))]
4561 .into_iter()
4562 .collect(),
4563 )
4564 .await,
4565 Err(TaskStoreError::RetentionLimitExceeded {
4566 kind: TaskRetentionLimitKind::PayloadBytes,
4567 ..
4568 })
4569 ));
4570 assert_eq!(store.usage(), before_response);
4571 assert_eq!(
4572 store
4573 .outstanding_input_requests(&other)
4574 .await
4575 .unwrap()
4576 .unwrap()
4577 .len(),
4578 1
4579 );
4580 assert_accounting(&store);
4581 }
4582
4583 #[tokio::test]
4584 async fn oversized_terminal_result_records_bounded_failure_and_wakes_waiter() {
4585 let store = store_with_limits(
4586 TaskRetentionLimits::unbounded()
4587 .max_payload_bytes(256)
4588 .max_retained_bytes(8 * 1024),
4589 );
4590 let id = working_task(&store, None).await;
4591 let waiting_store = store.clone();
4592 let waiting_id = id.clone();
4593 let mut waiter = tokio::spawn(async move {
4594 waiting_store
4595 .wait_for_completion(&waiting_id)
4596 .await
4597 .unwrap()
4598 .unwrap()
4599 });
4600 assert!(
4601 tokio::time::timeout(Duration::from_millis(20), &mut waiter)
4602 .await
4603 .is_err()
4604 );
4605
4606 let secret = "terminal-secret".repeat(1_024);
4607 let error = store
4608 .complete_task(&id, CallToolResult::text(&secret))
4609 .await
4610 .unwrap_err();
4611 assert!(matches!(
4612 error,
4613 TaskStoreError::RetentionLimitExceeded {
4614 kind: TaskRetentionLimitKind::PayloadBytes,
4615 limit: 256
4616 }
4617 ));
4618
4619 let (task, result, error) = tokio::time::timeout(Duration::from_secs(2), waiter)
4620 .await
4621 .expect("bounded failure did not wake completion waiter")
4622 .unwrap();
4623 assert_eq!(task.status, TaskStatus::Failed);
4624 assert_eq!(
4625 task.status_message.as_deref(),
4626 Some(RETENTION_FAILURE_STATUS)
4627 );
4628 assert!(result.is_none());
4629 assert_eq!(error.unwrap().message, RETENTION_FAILURE_MESSAGE);
4630 let data = store.state.data.read().unwrap();
4631 let stored = data.tasks.get(&id).unwrap();
4632 assert_eq!(stored.reserved_bytes, 0);
4633 assert!(!format!("{:?}", stored.task).contains("terminal-secret"));
4634 drop(data);
4635 assert_accounting(&store);
4636 }
4637
4638 #[tokio::test]
4639 async fn oversized_failure_and_cancel_reason_store_only_bounded_terminal_state() {
4640 let store = store_with_limits(
4641 TaskRetentionLimits::unbounded()
4642 .max_payload_bytes(128)
4643 .max_retained_bytes(8 * 1024),
4644 );
4645 let failed = working_task(&store, None).await;
4646 let secret = "diagnostic-secret".repeat(1_024);
4647 assert!(matches!(
4648 store
4649 .fail_task(&failed, JsonRpcError::internal_error(&secret))
4650 .await,
4651 Err(TaskStoreError::RetentionLimitExceeded { .. })
4652 ));
4653 let (task, _, error) = store.get_task_result(&failed).await.unwrap().unwrap();
4654 assert_eq!(task.status, TaskStatus::Failed);
4655 assert_eq!(error.unwrap().message, RETENTION_FAILURE_MESSAGE);
4656
4657 let cancelled = working_task(&store, None).await;
4658 let object = store
4659 .cancel_task(&cancelled, Some(&secret))
4660 .await
4661 .unwrap()
4662 .unwrap();
4663 assert_eq!(object.status, TaskStatus::Cancelled);
4664 assert_eq!(
4665 object.status_message.as_deref(),
4666 Some("Task cancelled: retention limit exceeded")
4667 );
4668 let data = store.state.data.read().unwrap();
4669 assert!(
4670 !format!("{:?}", data.tasks.get(&failed).unwrap().task).contains("diagnostic-secret")
4671 );
4672 assert!(
4673 !format!("{:?}", data.tasks.get(&cancelled).unwrap().task)
4674 .contains("diagnostic-secret")
4675 );
4676 drop(data);
4677 assert_accounting(&store);
4678 }
4679
4680 fn working_charge(tool: &str, arguments: serde_json::Value) -> usize {
4681 let limits = TaskRetentionLimits::unbounded();
4682 let task = Task::new(
4683 "prototype".to_string(),
4684 tool.to_string(),
4685 arguments,
4686 60_000,
4687 None,
4688 );
4689 let stored = prepare_stored_task(task, limits).unwrap();
4690 stored.retained_bytes + stored.reserved_bytes
4691 }
4692
4693 #[tokio::test]
4694 async fn aggregate_capacity_recovers_after_deletion_and_expiry() {
4695 let charge = working_charge("tool", serde_json::json!({}));
4696 let limits = TaskRetentionLimits::unbounded()
4697 .max_tasks(2)
4698 .max_retained_bytes(charge);
4699 let store = store_with_limits(limits);
4700 let first = working_task(&store, None).await;
4701 assert_eq!(store.usage().charged_bytes(), charge);
4702 assert!(matches!(
4703 store
4704 .create_task("tool", serde_json::json!({}), None, None)
4705 .await,
4706 Err(TaskStoreError::RetentionLimitExceeded {
4707 kind: TaskRetentionLimitKind::AggregateBytes,
4708 limit
4709 }) if limit == charge
4710 ));
4711 assert_accounting(&store);
4712 assert!(store.discard_task(&first).await.unwrap());
4713 assert_eq!(store.usage(), TaskStoreUsage::default());
4714
4715 let expired = working_task(&store, None).await;
4716 let before_expiry = store.usage();
4717 assert!(matches!(
4718 store
4719 .create_task("tool", serde_json::json!({}), None, None)
4720 .await,
4721 Err(TaskStoreError::RetentionLimitExceeded {
4722 kind: TaskRetentionLimitKind::AggregateBytes,
4723 limit
4724 }) if limit == charge
4725 ));
4726 assert_accounting(&store);
4727 assert!(store.set_ttl(&expired, 0).await.unwrap());
4728 let tombstone = store.usage();
4729 assert_eq!(tombstone.task_count, 1);
4730 assert_eq!(tombstone.reserved_bytes, 0);
4731 assert!(tombstone.retained_bytes < before_expiry.retained_bytes);
4732 assert!(matches!(
4733 store.task_presence(&expired).await.unwrap(),
4734 TaskPresence::Expired { .. }
4735 ));
4736 assert_accounting(&store);
4737
4738 let replacement = working_task(&store, None).await;
4739 assert_ne!(replacement, expired);
4740 assert!(matches!(
4741 store.task_presence(&expired).await.unwrap(),
4742 TaskPresence::Missing
4743 ));
4744 assert_eq!(store.usage().task_count, 1);
4745 assert_eq!(store.usage().charged_bytes(), charge);
4746 assert_accounting(&store);
4747 }
4748
4749 #[tokio::test]
4750 async fn expiry_accounting_allows_scrubbed_encoding_to_grow() {
4751 let store = MemoryTaskStore::with_config_and_retention(
4752 MemoryTaskStoreConfig::default().cleanup_interval(Duration::from_secs(60)),
4753 TaskRetentionLimits::unbounded(),
4754 );
4755 let (id, _) = store
4756 .create_task("x", serde_json::Value::Null, None, None)
4757 .await
4758 .unwrap();
4759 assert!(
4760 store
4761 .set_status(&id, TaskStatus::Working, Some(""))
4762 .await
4763 .unwrap()
4764 );
4765 let before_expiry = store.usage();
4766
4767 assert!(store.set_ttl(&id, 0).await.unwrap());
4768 let after_expiry = store.usage();
4769 assert_eq!(after_expiry.task_count, 1);
4770 assert_eq!(after_expiry.reserved_bytes, 0);
4771 assert_eq!(
4772 after_expiry.retained_bytes,
4773 before_expiry.retained_bytes + 1,
4774 "the empty status encodes two bytes smaller than None, while the one-byte tool name is scrubbed"
4775 );
4776 assert!(matches!(
4777 store.task_presence(&id).await.unwrap(),
4778 TaskPresence::Expired { .. }
4779 ));
4780 assert_accounting(&store);
4781 }
4782
4783 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
4784 async fn concurrent_completions_atomically_enforce_aggregate_limit() {
4785 let result_text = "r".repeat(2_048);
4786 let unbounded = TaskRetentionLimits::unbounded();
4787 let working = prepare_stored_task(
4788 Task::new(
4789 "prototype".to_string(),
4790 "tool".to_string(),
4791 serde_json::json!({}),
4792 60_000,
4793 None,
4794 ),
4795 unbounded,
4796 )
4797 .unwrap();
4798 let mut completed = Task::new(
4799 "prototype".to_string(),
4800 "tool".to_string(),
4801 serde_json::json!({}),
4802 60_000,
4803 None,
4804 );
4805 completed.status = TaskStatus::Completed;
4806 completed.status_message = Some("Task completed".to_string());
4807 completed.result = Some(CallToolResult::text(&result_text));
4808 completed.completed_at = Some(Instant::now());
4809 let completed = prepare_stored_task(completed, unbounded).unwrap();
4810 let working_charge = working.retained_bytes + working.reserved_bytes;
4811 assert!(completed.retained_bytes > working_charge);
4812 let cap = completed.retained_bytes + working_charge;
4813 let store = store_with_limits(
4814 TaskRetentionLimits::unbounded()
4815 .max_tasks(2)
4816 .max_payload_bytes(4 * 1024)
4817 .max_retained_bytes(cap),
4818 );
4819 let first = working_task(&store, None).await;
4820 let second = working_task(&store, None).await;
4821 let barrier = Arc::new(tokio::sync::Barrier::new(3));
4822 let mut completions = Vec::new();
4823 for id in [first.clone(), second.clone()] {
4824 let store = store.clone();
4825 let barrier = barrier.clone();
4826 let result_text = result_text.clone();
4827 completions.push(tokio::spawn(async move {
4828 barrier.wait().await;
4829 store
4830 .complete_task(&id, CallToolResult::text(result_text))
4831 .await
4832 }));
4833 }
4834 barrier.wait().await;
4835
4836 let mut completed_count = 0;
4837 let mut rejected_count = 0;
4838 for completion in completions {
4839 match completion.await.unwrap() {
4840 Ok(true) => completed_count += 1,
4841 Err(TaskStoreError::RetentionLimitExceeded {
4842 kind: TaskRetentionLimitKind::AggregateBytes,
4843 limit,
4844 }) if limit == cap => rejected_count += 1,
4845 other => panic!("unexpected completion outcome: {other:?}"),
4846 }
4847 }
4848 assert_eq!((completed_count, rejected_count), (1, 1));
4849 let states = [
4850 store.get_task(&first).await.unwrap().unwrap().status,
4851 store.get_task(&second).await.unwrap().unwrap().status,
4852 ];
4853 assert_eq!(
4854 states
4855 .iter()
4856 .filter(|status| **status == TaskStatus::Completed)
4857 .count(),
4858 1
4859 );
4860 assert_eq!(
4861 states
4862 .iter()
4863 .filter(|status| **status == TaskStatus::Failed)
4864 .count(),
4865 1
4866 );
4867 assert_accounting(&store);
4868 }
4869}