use std::collections::HashMap;
use std::io;
use std::path::{Path, PathBuf};
use chrono::{DateTime, Duration, Utc};
use rusty_agent_runtime::durable::{
classify_retry_with_policy, ErrorClass, ResolvedRetryParameters, RetryDecision,
};
use rusty_agent_runtime::llm::Usage;
use rusty_agent_runtime::record::ExecutorPolicy;
use rusty_agent_runtime::record::{Effect, EffectReceipt};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
pub(crate) const DEFAULT_POOL: &str = "default";
pub(crate) const MEMORY_CONSOLIDATION_KIND: &str = "memory_consolidation";
pub(crate) const DEFAULT_MAX_ATTEMPTS: u32 = 3;
pub(crate) const MAX_ATTEMPTS_LIMIT: u32 = 100;
pub(crate) const MIN_LEASE_MS: u64 = 100;
pub(crate) const MAX_LEASE_MS: u64 = 3_600_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub(crate) enum TaskStatus {
Queued,
Leased,
Failed,
Completed,
Dead,
Cancelled,
}
impl TaskStatus {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Queued => "queued",
Self::Leased => "leased",
Self::Failed => "failed",
Self::Completed => "completed",
Self::Dead => "dead",
Self::Cancelled => "cancelled",
}
}
pub(crate) fn parse(s: &str) -> Option<Self> {
match s {
"queued" => Some(Self::Queued),
"leased" => Some(Self::Leased),
"failed" => Some(Self::Failed),
"completed" => Some(Self::Completed),
"dead" => Some(Self::Dead),
"cancelled" => Some(Self::Cancelled),
_ => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct TaskLease {
pub owner: String,
pub expires_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct TaskRecord {
pub task_id: String,
pub tenant: String,
pub kind: String,
pub payload: Value,
pub pool: String,
#[serde(default)]
pub recipient: Option<String>,
pub status: TaskStatus,
pub attempt: u32,
pub max_attempts: u32,
#[serde(default)]
pub lease: Option<TaskLease>,
#[serde(default)]
pub error_class: Option<ErrorClass>,
#[serde(default)]
pub effect: Option<Effect>,
#[serde(default)]
pub last_error: Option<String>,
#[serde(default)]
pub idempotency_key: Option<String>,
#[serde(default)]
pub result: Option<Value>,
#[serde(default)]
pub receipt: Option<EffectReceipt>,
#[serde(default)]
pub tokens: Option<Usage>,
#[serde(default)]
pub cost_usd: Option<f64>,
#[serde(default)]
pub run_id: Option<String>,
#[serde(default)]
pub thread_id: Option<String>,
#[serde(default)]
pub parent: Option<String>,
#[serde(default)]
pub cancel_requested: bool,
#[serde(default)]
pub deadline: Option<DateTime<Utc>>,
#[serde(default)]
pub worker_version: Option<String>,
#[serde(default)]
pub next_attempt_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
pub(crate) struct NewTask {
pub task_id: String,
pub tenant: String,
pub kind: String,
pub payload: Value,
pub pool: String,
pub recipient: Option<String>,
pub max_attempts: u32,
pub idempotency_key: Option<String>,
pub effect: Option<Effect>,
pub run_id: Option<String>,
pub thread_id: Option<String>,
pub deadline: Option<DateTime<Utc>>,
pub parent: Option<String>,
pub worker_version: Option<String>,
}
pub(crate) struct FailureReport {
pub error_class: ErrorClass,
pub message: String,
pub retryable: bool,
pub cost: SettlementCost,
pub retry: ResolvedRetryParameters,
}
pub(crate) struct CompletionReport {
pub result: Value,
pub receipt: Option<EffectReceipt>,
pub cost: SettlementCost,
}
#[derive(Debug, Clone, Copy, Default, PartialEq)]
pub(crate) struct SettlementCost {
pub tokens: Option<Usage>,
pub cost_usd: Option<f64>,
}
impl SettlementCost {
#[allow(dead_code)]
pub(crate) const NONE: Self = Self {
tokens: None,
cost_usd: None,
};
}
impl TaskRecord {
pub(crate) fn new(new: NewTask, now: DateTime<Utc>) -> Self {
let NewTask {
task_id,
tenant,
kind,
payload,
pool,
recipient,
max_attempts,
idempotency_key,
effect,
run_id,
thread_id,
deadline,
worker_version,
parent,
} = new;
Self {
task_id,
tenant,
kind,
payload,
pool,
recipient,
status: TaskStatus::Queued,
attempt: 0,
max_attempts,
lease: None,
error_class: None,
effect,
last_error: None,
idempotency_key,
result: None,
receipt: None,
tokens: None,
cost_usd: None,
run_id,
thread_id,
parent,
cancel_requested: false,
deadline,
worker_version,
next_attempt_at: None,
created_at: now,
updated_at: now,
}
}
pub(crate) fn claimable_at(&self, now: DateTime<Utc>) -> bool {
match self.status {
TaskStatus::Queued => true,
TaskStatus::Failed => self.next_attempt_at.is_some_and(|at| at <= now),
TaskStatus::Leased => self.lease.as_ref().is_some_and(|l| l.expires_at <= now),
TaskStatus::Completed | TaskStatus::Dead | TaskStatus::Cancelled => false,
}
}
pub(crate) fn matches_worker_version(&self, worker_version: Option<&str>) -> bool {
self.worker_version
.as_deref()
.is_none_or(|pin| Some(pin) == worker_version)
}
pub(crate) fn is_terminal(&self) -> bool {
match self.status {
TaskStatus::Completed | TaskStatus::Dead | TaskStatus::Cancelled => true,
TaskStatus::Failed => self.next_attempt_at.is_none(),
TaskStatus::Queued | TaskStatus::Leased => false,
}
}
pub(crate) fn leased_to(&self, worker_id: &str) -> bool {
self.status == TaskStatus::Leased
&& self.lease.as_ref().is_some_and(|l| l.owner == worker_id)
}
pub(crate) fn claim(&mut self, worker_id: &str, lease_ms: u64, now: DateTime<Utc>) {
self.status = TaskStatus::Leased;
self.attempt = self.attempt.saturating_add(1);
self.lease = Some(TaskLease {
owner: worker_id.to_string(),
expires_at: now + lease_duration(lease_ms),
});
self.next_attempt_at = None;
self.updated_at = now;
}
pub(crate) fn renew_lease(&mut self, lease_ms: u64, now: DateTime<Utc>) {
if let Some(lease) = &mut self.lease {
lease.expires_at = now + lease_duration(lease_ms);
}
self.updated_at = now;
}
pub(crate) fn complete(
&mut self,
result: Value,
receipt: Option<EffectReceipt>,
cost: SettlementCost,
now: DateTime<Utc>,
) {
self.status = TaskStatus::Completed;
self.result = Some(result);
self.receipt = receipt;
self.tokens = cost.tokens;
self.cost_usd = cost.cost_usd;
self.lease = None;
self.next_attempt_at = None;
self.updated_at = now;
}
pub(crate) fn cancel(&mut self, now: DateTime<Utc>) -> Option<CancelTransition> {
if self.is_terminal() {
return None;
}
if self.status == TaskStatus::Leased {
self.cancel_requested = true;
self.updated_at = now;
Some(CancelTransition::Signalled)
} else {
self.apply_cancellation(now);
Some(CancelTransition::Cancelled)
}
}
pub(crate) fn apply_cancellation(&mut self, now: DateTime<Utc>) {
self.status = TaskStatus::Cancelled;
self.error_class = Some(ErrorClass::Cancelled);
self.lease = None;
self.next_attempt_at = None;
self.updated_at = now;
}
pub(crate) fn cancellation_due(&self, now: DateTime<Utc>) -> bool {
self.claimable_at(now) && (self.cancel_requested || self.deadline.is_some_and(|d| d <= now))
}
pub(crate) fn fail(
&mut self,
error_class: ErrorClass,
message: &str,
retryable: bool,
cost: SettlementCost,
retry: &ResolvedRetryParameters,
now: DateTime<Utc>,
) {
let effect = self.effect.unwrap_or(if retryable {
Effect::Idempotent
} else {
Effect::NonIdempotent
});
match classify_retry_with_policy(effect, error_class, self.attempt, retry, uniform()) {
RetryDecision::Retry { after_ms } => {
self.status = TaskStatus::Failed;
self.next_attempt_at =
Some(now + Duration::milliseconds(after_ms.min(i64::MAX as u64) as i64));
}
RetryDecision::Dead => {
self.status = TaskStatus::Dead;
self.next_attempt_at = None;
}
RetryDecision::Fail => {
self.status = if error_class == ErrorClass::Cancelled {
TaskStatus::Cancelled
} else {
TaskStatus::Failed
};
self.next_attempt_at = None;
}
}
self.error_class = Some(error_class);
self.last_error = Some(message.to_string());
self.tokens = cost.tokens;
self.cost_usd = cost.cost_usd;
self.lease = None;
self.updated_at = now;
}
pub(crate) fn wire(&self) -> Value {
json!({
"task_id": self.task_id,
"kind": self.kind,
"payload": self.payload,
"pool": self.pool,
"recipient": self.recipient,
"status": self.status.as_str(),
"attempt": self.attempt,
"max_attempts": self.max_attempts,
"error_class": self.error_class,
"effect": self.effect,
"last_error": self.last_error,
"idempotency_key": self.idempotency_key,
"result": self.result,
"receipt": self.receipt,
"tokens": self.tokens,
"cost_usd": self.cost_usd,
"run_id": self.run_id,
"thread_id": self.thread_id,
"parent": self.parent,
"cancel_requested": self.cancel_requested,
"deadline": self.deadline,
"worker_version": self.worker_version,
"lease": self.lease.as_ref().map(|lease| json!({
"owner": lease.owner,
"expires_at": lease.expires_at,
})),
"next_attempt_at": self.next_attempt_at,
"created_at": self.created_at,
"updated_at": self.updated_at,
})
}
}
fn lease_duration(lease_ms: u64) -> Duration {
Duration::milliseconds(lease_ms.min(i64::MAX as u64) as i64)
}
#[derive(Debug, Clone)]
pub(crate) enum MutationOutcome {
Applied(Box<TaskRecord>),
LeaseLost,
Unknown,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum CancelTransition {
Cancelled,
Signalled,
}
#[derive(Debug, Clone)]
pub(crate) enum CancelOutcome {
Applied(Box<TaskRecord>),
Terminal(TaskStatus),
Unknown,
}
#[derive(Debug, Default)]
pub(crate) struct RunCancellation {
pub cancelled: Vec<TaskRecord>,
pub signalled: Vec<TaskRecord>,
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct ClaimScope<'a> {
pub pools: &'a [String],
pub pool_limits: &'a HashMap<String, usize>,
pub worker_version: Option<&'a str>,
pub timeout_policy: &'a ExecutorPolicy,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) struct TaskUsage {
pub queued: u64,
pub in_flight: u64,
pub dlq: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct PoolStat {
pub pool: String,
pub queue_depth: u64,
pub leased: u64,
pub oldest_visible_at: Option<DateTime<Utc>>,
}
fn uniform() -> f64 {
let bytes = uuid::Uuid::new_v4().into_bytes();
let roll = u64::from_le_bytes(bytes[..8].try_into().expect("uuid is 16 bytes"));
(roll >> 11) as f64 / (1u64 << 53) as f64
}
fn parse_taxonomy<T: serde::de::DeserializeOwned>(
what: &str,
raw: &str,
expected: &str,
) -> Result<T, String> {
serde_json::from_value(Value::String(raw.to_string()))
.map_err(|_| format!("unknown `{what}` `{raw}` (expected {expected})"))
}
#[cfg_attr(not(feature = "postgres"), allow(dead_code))]
fn taxonomy_name<T: Serialize>(value: T) -> String {
match serde_json::to_value(value).expect("taxonomy serialization is infallible") {
Value::String(name) => name,
_ => unreachable!("taxonomy enums serialize to strings"),
}
}
pub(crate) fn parse_error_class(raw: &str) -> Result<ErrorClass, String> {
parse_taxonomy(
"error_class",
raw,
"transient|rate_limited|timeout|invalid_input|dependency_failure|resource_exhausted|cancelled|unknown",
)
}
#[cfg_attr(not(feature = "postgres"), allow(dead_code))]
pub(crate) fn error_class_name(class: ErrorClass) -> String {
taxonomy_name(class)
}
pub(crate) fn parse_effect(raw: &str) -> Result<Effect, String> {
parse_taxonomy(
"effect",
raw,
"pure|read_only|idempotent|compensatable|non_idempotent",
)
}
#[cfg_attr(not(feature = "postgres"), allow(dead_code))]
pub(crate) fn effect_name(effect: Effect) -> String {
taxonomy_name(effect)
}
pub(crate) fn validate_label(what: &str, value: &str, max_len: usize) -> Result<(), String> {
if value.trim().is_empty() || value.len() > max_len {
return Err(format!("`{what}` must be non-empty and <= {max_len} chars"));
}
Ok(())
}
pub(crate) fn validate_pool(pool: &str) -> Result<(), String> {
let ok = !pool.is_empty()
&& pool.len() <= 128
&& pool
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.'));
if ok {
Ok(())
} else {
Err("`pool` must match [A-Za-z0-9._-] and be 1..=128 chars".to_string())
}
}
pub(crate) fn validate_lease_ms(lease_ms: u64) -> Result<(), String> {
if (MIN_LEASE_MS..=MAX_LEASE_MS).contains(&lease_ms) {
Ok(())
} else {
Err(format!(
"`lease_ms` must be within {MIN_LEASE_MS}..={MAX_LEASE_MS}"
))
}
}
pub(crate) fn dir(root: &Path) -> PathBuf {
root.join("tasks")
}
pub(crate) async fn persist(root: &Path, record: &TaskRecord) -> io::Result<()> {
let dir = dir(root);
tokio::fs::create_dir_all(&dir).await?;
let bytes = serde_json::to_vec_pretty(record)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let path = dir.join(format!("{}.json", record.task_id));
let tmp = dir.join(format!("{}.tmp", record.task_id));
tokio::fs::write(&tmp, bytes).await?;
tokio::fs::rename(&tmp, path).await
}
pub(crate) fn load(root: &Path) -> HashMap<String, TaskRecord> {
let mut out = HashMap::new();
let Ok(entries) = std::fs::read_dir(dir(root)) else {
return out;
};
for entry in entries.flatten() {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let parsed = std::fs::read_to_string(&path)
.ok()
.and_then(|raw| serde_json::from_str::<TaskRecord>(&raw).ok());
match parsed {
Some(record) => {
out.insert(record.task_id.clone(), record);
}
None => {
tracing::warn!(path = %path.display(), "skipping unreadable task file")
}
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn record() -> TaskRecord {
TaskRecord::new(
NewTask {
task_id: "task-1".to_string(),
tenant: "acme".to_string(),
kind: "send_email".to_string(),
payload: json!({"to": "a@b.c"}),
pool: DEFAULT_POOL.to_string(),
recipient: None,
max_attempts: DEFAULT_MAX_ATTEMPTS,
idempotency_key: None,
effect: None,
run_id: None,
thread_id: None,
deadline: None,
worker_version: None,
parent: None,
},
Utc::now(),
)
}
fn floor() -> ResolvedRetryParameters {
ResolvedRetryParameters::floor(DEFAULT_MAX_ATTEMPTS)
}
#[test]
fn status_wire_spellings_round_trip() {
for (status, s) in [
(TaskStatus::Queued, "queued"),
(TaskStatus::Leased, "leased"),
(TaskStatus::Failed, "failed"),
(TaskStatus::Completed, "completed"),
(TaskStatus::Dead, "dead"),
(TaskStatus::Cancelled, "cancelled"),
] {
assert_eq!(status.as_str(), s);
assert_eq!(TaskStatus::parse(s), Some(status));
assert_eq!(serde_json::to_value(status).unwrap(), json!(s));
assert_eq!(
serde_json::from_value::<TaskStatus>(json!(s)).unwrap(),
status
);
}
assert_eq!(TaskStatus::parse("zombie"), None);
}
#[test]
fn fail_with_retryable_false_is_terminal_but_not_dead_lettered() {
let mut task = record();
let t0 = Utc::now();
task.claim("w-1", 60_000, t0);
task.fail(
ErrorClass::Timeout,
"charged twice maybe",
false,
SettlementCost::NONE,
&floor(),
t0,
);
assert_eq!(task.status, TaskStatus::Failed);
assert_eq!(task.next_attempt_at, None);
assert!(!task.claimable_at(t0 + Duration::days(365)));
assert_eq!(task.error_class, Some(ErrorClass::Timeout));
}
#[test]
fn fail_with_non_retryable_class_is_terminal_despite_retryable_flag() {
let mut task = record();
let t0 = Utc::now();
task.claim("w-1", 60_000, t0);
task.fail(
ErrorClass::InvalidInput,
"bad schema",
true,
SettlementCost::NONE,
&floor(),
t0,
);
assert_eq!(task.status, TaskStatus::Failed);
assert_eq!(task.next_attempt_at, None);
}
#[test]
fn fail_retryable_within_budget_requeues_inside_the_jitter_window() {
for attempt in 1..=8u32 {
for _ in 0..64 {
let mut task = record();
task.max_attempts = MAX_ATTEMPTS_LIMIT;
let t0 = Utc::now();
task.attempt = attempt - 1;
task.claim("w-1", 60_000, t0);
task.fail(
ErrorClass::Transient,
"hiccup",
true,
SettlementCost::NONE,
&ResolvedRetryParameters::floor(MAX_ATTEMPTS_LIMIT),
t0,
);
assert_eq!(task.status, TaskStatus::Failed);
let at = task.next_attempt_at.expect("retry scheduled");
let bound = (1_000u64 << (attempt - 1)).min(300_000) as i64;
let delay = (at - t0).num_milliseconds();
assert!(
(0..=bound).contains(&delay),
"delay {delay} ms outside [0, {bound}] for attempt {attempt}"
);
}
}
}
#[test]
fn fail_at_the_attempt_ceiling_dead_letters() {
let mut task = record(); let t0 = Utc::now();
task.claim("w-1", 60_000, t0);
task.attempt = 3;
task.fail(
ErrorClass::Unknown,
"third strike",
true,
SettlementCost::NONE,
&floor(),
t0,
);
assert_eq!(task.status, TaskStatus::Dead);
assert_eq!(task.next_attempt_at, None);
assert!(!task.claimable_at(t0 + Duration::days(365)));
}
#[test]
fn declared_effect_outranks_the_workers_retryable_flag() {
let mut task = record();
task.effect = Some(Effect::NonIdempotent);
let t0 = Utc::now();
task.claim("w-1", 60_000, t0);
task.fail(
ErrorClass::Timeout,
"maybe it fired",
true,
SettlementCost::NONE,
&floor(),
t0,
);
assert_eq!(task.status, TaskStatus::Failed);
assert_eq!(task.next_attempt_at, None, "effect gate: fail outright");
let mut task = record();
task.effect = Some(Effect::Idempotent);
task.claim("w-1", 60_000, t0);
task.fail(
ErrorClass::Transient,
"hiccup",
false,
SettlementCost::NONE,
&floor(),
t0,
);
assert_eq!(task.status, TaskStatus::Failed);
assert!(
task.next_attempt_at.is_some(),
"declared idempotent retries"
);
}
#[test]
fn error_class_names_round_trip_through_the_serde_contract() {
for (class, name) in [
(ErrorClass::Transient, "transient"),
(ErrorClass::RateLimited, "rate_limited"),
(ErrorClass::Timeout, "timeout"),
(ErrorClass::InvalidInput, "invalid_input"),
(ErrorClass::DependencyFailure, "dependency_failure"),
(ErrorClass::ResourceExhausted, "resource_exhausted"),
(ErrorClass::Cancelled, "cancelled"),
(ErrorClass::Unknown, "unknown"),
] {
assert_eq!(error_class_name(class), name);
assert_eq!(parse_error_class(name).unwrap(), class);
}
assert!(parse_error_class("bug").is_err());
assert!(parse_error_class("").is_err());
assert!(
parse_error_class("Transient").is_err(),
"wire names are snake_case"
);
}
#[test]
fn effect_names_round_trip_through_the_serde_contract() {
for (effect, name) in [
(Effect::Pure, "pure"),
(Effect::ReadOnly, "read_only"),
(Effect::Idempotent, "idempotent"),
(Effect::Compensatable, "compensatable"),
(Effect::NonIdempotent, "non_idempotent"),
] {
assert_eq!(effect_name(effect), name);
assert_eq!(parse_effect(name).unwrap(), effect);
}
assert!(parse_effect("side_effecty").is_err());
assert!(
parse_effect("Idempotent").is_err(),
"wire names are snake_case"
);
}
#[test]
fn uniform_stays_inside_unit_interval() {
for _ in 0..256 {
let u = uniform();
assert!((0.0..1.0).contains(&u), "uniform sample {u} outside [0, 1)");
}
}
#[test]
fn lifecycle_claim_heartbeat_complete() {
let mut task = record();
let t0 = Utc::now();
assert!(task.claimable_at(t0));
assert!(!task.leased_to("w-1"));
task.claim("w-1", 60_000, t0);
assert_eq!(task.status, TaskStatus::Leased);
assert_eq!(task.attempt, 1);
assert!(task.leased_to("w-1"));
assert!(!task.leased_to("w-2"));
assert!(!task.claimable_at(t0 + Duration::seconds(30)));
assert!(task.claimable_at(t0 + Duration::seconds(61)));
task.renew_lease(60_000, t0);
let expires = task.lease.as_ref().unwrap().expires_at;
assert_eq!(expires, t0 + Duration::seconds(60));
task.complete(json!({"ok": true}), None, SettlementCost::NONE, t0);
assert_eq!(task.status, TaskStatus::Completed);
assert_eq!(task.result, Some(json!({"ok": true})));
assert!(task.lease.is_none());
assert!(!task.claimable_at(t0 + Duration::days(365)));
}
#[test]
fn lifecycle_failed_attempt_schedules_retry_then_dead_letters() {
let mut task = record();
let t0 = Utc::now();
task.claim("w-1", 60_000, t0);
task.fail(
ErrorClass::Timeout,
"upstream timed out",
true,
SettlementCost::NONE,
&floor(),
t0,
);
assert_eq!(task.status, TaskStatus::Failed);
assert_eq!(task.error_class, Some(ErrorClass::Timeout));
assert_eq!(task.last_error.as_deref(), Some("upstream timed out"));
assert!(task.lease.is_none());
let at = task.next_attempt_at.expect("retry scheduled");
assert!(!task.claimable_at(t0));
assert!(task.claimable_at(at));
task.claim("w-1", 60_000, at);
assert_eq!(task.attempt, 2);
assert!(task.next_attempt_at.is_none(), "claim clears the schedule");
task.fail(
ErrorClass::Timeout,
"again",
true,
SettlementCost::NONE,
&floor(),
at,
);
assert_eq!(task.status, TaskStatus::Failed);
let at = task.next_attempt_at.unwrap();
task.claim("w-2", 60_000, at);
assert_eq!(task.attempt, 3);
task.fail(
ErrorClass::Timeout,
"third strike",
true,
SettlementCost::NONE,
&floor(),
at,
);
assert_eq!(task.status, TaskStatus::Dead);
assert!(task.next_attempt_at.is_none());
assert!(!task.claimable_at(at + Duration::days(365)));
}
#[test]
fn expired_lease_reclaim_counts_a_new_attempt() {
let mut task = record();
let t0 = Utc::now();
task.claim("w-1", 1_000, t0);
let t1 = t0 + Duration::seconds(2);
assert!(task.claimable_at(t1));
assert!(task.leased_to("w-1"));
task.claim("w-2", 1_000, t1);
assert_eq!(task.attempt, 2);
assert!(task.leased_to("w-2"));
assert!(!task.leased_to("w-1"), "the lost lease no longer settles");
}
#[test]
fn cancel_queued_task_is_terminal_immediately() {
let mut task = record();
let t0 = Utc::now();
assert_eq!(task.cancel(t0), Some(CancelTransition::Cancelled));
assert_eq!(task.status, TaskStatus::Cancelled);
assert_eq!(task.error_class, Some(ErrorClass::Cancelled));
assert!(task.next_attempt_at.is_none());
assert!(task.lease.is_none());
assert!(!task.cancel_requested, "no holder to signal");
assert!(task.is_terminal());
assert!(!task.claimable_at(t0 + Duration::days(365)));
assert_eq!(task.cancel(t0), None);
}
#[test]
fn cancel_retry_scheduled_task_is_terminal_immediately() {
let mut task = record();
let t0 = Utc::now();
task.claim("w-1", 60_000, t0);
task.fail(
ErrorClass::Transient,
"hiccup",
true,
SettlementCost::NONE,
&floor(),
t0,
);
assert_eq!(task.status, TaskStatus::Failed);
assert!(task.next_attempt_at.is_some(), "a retry is outstanding");
assert!(!task.is_terminal());
assert_eq!(task.cancel(t0), Some(CancelTransition::Cancelled));
assert_eq!(task.status, TaskStatus::Cancelled);
assert!(task.next_attempt_at.is_none());
}
#[test]
fn cancel_leased_task_signals_the_holder_and_keeps_its_lease() {
let mut task = record();
let t0 = Utc::now();
task.claim("w-1", 60_000, t0);
assert_eq!(task.cancel(t0), Some(CancelTransition::Signalled));
assert_eq!(task.status, TaskStatus::Leased);
assert!(task.cancel_requested);
assert!(task.leased_to("w-1"));
assert_eq!(task.cancel(t0), Some(CancelTransition::Signalled));
task.fail(
ErrorClass::Cancelled,
"cancelled by control plane",
false,
SettlementCost::NONE,
&floor(),
t0,
);
assert_eq!(task.status, TaskStatus::Cancelled);
assert_eq!(task.error_class, Some(ErrorClass::Cancelled));
assert!(!task.claimable_at(t0 + Duration::days(365)));
}
#[test]
fn cancel_terminal_task_is_refused() {
let t0 = Utc::now();
let mut completed = record();
completed.claim("w-1", 60_000, t0);
completed.complete(json!({"ok": true}), None, SettlementCost::NONE, t0);
let mut dead = record();
dead.claim("w-1", 60_000, t0);
dead.attempt = dead.max_attempts;
dead.fail(
ErrorClass::Unknown,
"third strike",
true,
SettlementCost::NONE,
&floor(),
t0,
);
let mut failed_outright = record();
failed_outright.claim("w-1", 60_000, t0);
failed_outright.fail(
ErrorClass::InvalidInput,
"bad schema",
true,
SettlementCost::NONE,
&floor(),
t0,
);
let mut cancelled = record();
cancelled.cancel(t0);
for task in [
&mut completed,
&mut dead,
&mut failed_outright,
&mut cancelled,
] {
let before = task.status;
assert!(task.is_terminal(), "{before:?} is terminal");
assert_eq!(task.cancel(t0), None, "{before:?} accepted a cancel");
assert_eq!(task.status, before, "cancel changed a terminal task");
}
}
#[test]
fn fail_with_cancelled_class_lands_cancelled_not_failed_or_dead() {
let mut task = record();
let t0 = Utc::now();
task.claim("w-1", 60_000, t0);
task.fail(
ErrorClass::Cancelled,
"interrupted",
true,
SettlementCost::NONE,
&floor(),
t0,
);
assert_eq!(task.status, TaskStatus::Cancelled);
assert!(task.next_attempt_at.is_none());
assert!(!task.claimable_at(t0 + Duration::days(365)));
}
#[test]
fn cancellation_due_covers_the_unanswered_and_the_expired() {
let t0 = Utc::now();
let mut task = record();
task.claim("w-1", 60_000, t0);
task.cancel(t0);
assert!(!task.cancellation_due(t0));
assert!(task.cancellation_due(t0 + Duration::seconds(61)));
let mut task = record();
task.deadline = Some(t0 - Duration::seconds(1));
assert!(task.cancellation_due(t0));
let mut task = record();
task.deadline = Some(t0 + Duration::seconds(60));
assert!(!task.cancellation_due(t0));
assert!(!record().cancellation_due(t0));
}
#[test]
fn apply_cancellation_clears_everything_that_could_requeue() {
let mut task = record();
let t0 = Utc::now();
task.claim("w-1", 1_000, t0);
task.cancel(t0);
task.apply_cancellation(t0 + Duration::seconds(2));
assert_eq!(task.status, TaskStatus::Cancelled);
assert!(task.lease.is_none(), "no lease holder survives");
assert!(task.next_attempt_at.is_none());
assert!(!task.leased_to("w-1"));
}
#[test]
fn wire_omits_tenant_and_nests_the_lease() {
let mut task = record();
task.claim("w-1", 60_000, Utc::now());
let wire = task.wire();
assert!(wire.get("tenant").is_none());
assert_eq!(wire["task_id"], json!("task-1"));
assert_eq!(wire["kind"], json!("send_email"));
assert_eq!(wire["status"], json!("leased"));
assert_eq!(wire["lease"]["owner"], json!("w-1"));
assert!(wire["lease"]["expires_at"].is_string());
assert_eq!(wire["idempotency_key"], Value::Null);
assert_eq!(wire["effect"], Value::Null);
assert_eq!(wire["run_id"], Value::Null);
assert_eq!(wire["cancel_requested"], json!(false));
assert_eq!(wire["deadline"], Value::Null);
assert_eq!(wire["worker_version"], Value::Null);
assert_eq!(wire["next_attempt_at"], Value::Null);
assert_eq!(wire["recipient"], Value::Null);
}
#[test]
fn wire_carries_the_recipient_when_addressed() {
let mut task = record();
task.recipient = Some("agent:researcher-7".to_string());
let wire = task.wire();
assert_eq!(wire["recipient"], json!("agent:researcher-7"));
}
#[test]
fn record_serde_round_trip() {
let mut task = record();
task.idempotency_key = Some("key-1".to_string());
task.run_id = Some("run-9".to_string());
task.deadline = DateTime::<Utc>::from_timestamp_millis(1_800_000_000_000);
task.claim("w-1", 60_000, Utc::now());
task.fail(
ErrorClass::Unknown,
"it broke",
true,
SettlementCost::NONE,
&floor(),
Utc::now(),
);
let raw = serde_json::to_string(&task).unwrap();
let back: TaskRecord = serde_json::from_str(&raw).unwrap();
assert_eq!(back.task_id, task.task_id);
assert_eq!(back.tenant, task.tenant);
assert_eq!(back.status, TaskStatus::Failed);
assert_eq!(back.attempt, 1);
assert_eq!(back.idempotency_key.as_deref(), Some("key-1"));
assert_eq!(back.error_class, Some(ErrorClass::Unknown));
assert!(raw.contains("\"error_class\":\"unknown\""));
assert_eq!(back.next_attempt_at, task.next_attempt_at);
assert_eq!(back.run_id.as_deref(), Some("run-9"));
assert_eq!(back.deadline, task.deadline);
assert!(!back.cancel_requested);
let legacy = json!({
"task_id": "t", "tenant": "default", "kind": "k", "payload": null,
"pool": "default", "status": "queued", "attempt": 0,
"max_attempts": 3, "created_at": Utc::now(), "updated_at": Utc::now(),
});
let back: TaskRecord = serde_json::from_value(legacy).unwrap();
assert_eq!(back.status, TaskStatus::Queued);
assert!(back.lease.is_none() && back.result.is_none() && back.run_id.is_none());
assert!(back.effect.is_none() && back.error_class.is_none());
assert!(!back.cancel_requested && back.deadline.is_none());
assert!(
back.worker_version.is_none(),
"wave-3 pin defaults to unpinned"
);
assert!(
back.recipient.is_none(),
"R0.7 recipient addressing defaults to ordinary pool work"
);
}
#[test]
fn worker_version_pin_matches_only_the_exact_string() {
let mut pinned = record();
pinned.worker_version = Some("activity-worker/1.4.0".to_string());
assert!(pinned.matches_worker_version(Some("activity-worker/1.4.0")));
assert!(!pinned.matches_worker_version(Some("activity-worker/1.5.0")));
assert!(
!pinned.matches_worker_version(None),
"a worker advertising no version cannot take a pinned task"
);
let unpinned = record();
assert!(unpinned.matches_worker_version(None));
assert!(unpinned.matches_worker_version(Some("activity-worker/1.4.0")));
}
#[test]
fn validation_bounds() {
assert!(validate_label("kind", "send_email", 256).is_ok());
assert!(validate_label("kind", " ", 256).is_err());
assert!(validate_label("kind", &"x".repeat(257), 256).is_err());
assert!(validate_pool("default").is_ok());
assert!(validate_pool("gpu-workers.eu-west").is_ok());
assert!(validate_pool("").is_err());
assert!(validate_pool("bad/pool").is_err());
assert!(validate_lease_ms(100).is_ok());
assert!(validate_lease_ms(3_600_000).is_ok());
assert!(validate_lease_ms(99).is_err());
assert!(validate_lease_ms(3_600_001).is_err());
}
#[tokio::test]
async fn persist_then_load_round_trips() {
let root = std::env::temp_dir().join(format!("rusty-tasks-test-{}", uuid::Uuid::new_v4()));
let mut task = record();
task.claim("w-1", 60_000, Utc::now());
persist(&root, &task).await.unwrap();
let loaded = load(&root);
assert_eq!(loaded.len(), 1);
assert_eq!(loaded["task-1"].status, TaskStatus::Leased);
assert!(loaded["task-1"].leased_to("w-1"));
task.complete(json!(1), None, SettlementCost::NONE, Utc::now());
persist(&root, &task).await.unwrap();
std::fs::write(dir(&root).join("corrupt.json"), b"{nope").unwrap();
let loaded = load(&root);
assert_eq!(loaded.len(), 1);
assert_eq!(loaded["task-1"].status, TaskStatus::Completed);
let _ = std::fs::remove_dir_all(root);
}
#[test]
fn settlement_cost_is_stored_on_complete_and_fail() {
let usage = Usage {
prompt_tokens: 120,
completion_tokens: 30,
total_tokens: 150,
};
let cost = SettlementCost {
tokens: Some(usage),
cost_usd: Some(0.0042),
};
let mut task = record();
let t0 = Utc::now();
task.claim("w-1", 60_000, t0);
task.complete(json!({"ok": true}), None, cost, t0);
assert_eq!(task.tokens, Some(usage));
assert_eq!(task.cost_usd, Some(0.0042));
assert_eq!(task.wire()["tokens"]["total_tokens"], json!(150));
assert_eq!(task.wire()["cost_usd"], json!(0.0042));
let mut task = record();
task.claim("w-1", 60_000, t0);
task.fail(ErrorClass::Unknown, "gave up", false, cost, &floor(), t0);
assert_eq!(task.tokens, Some(usage));
assert_eq!(task.cost_usd, Some(0.0042));
let mut task = record();
task.claim("w-1", 60_000, t0);
task.complete(json!(1), None, SettlementCost::NONE, t0);
assert_eq!(task.tokens, None);
assert!(task.wire()["tokens"].is_null());
}
#[test]
fn pre_wave3_records_deserialize_with_defaults() {
let legacy = json!({
"task_id": "task-1",
"tenant": "acme",
"kind": "send_email",
"payload": {"to": "a@b.c"},
"pool": "default",
"status": "queued",
"attempt": 0,
"max_attempts": 3,
"cancel_requested": false,
"created_at": "2027-01-15T08:00:00Z",
"updated_at": "2027-01-15T08:00:00Z",
});
let record: TaskRecord = serde_json::from_value(legacy).unwrap();
assert_eq!(record.tokens, None);
assert_eq!(record.cost_usd, None);
assert_eq!(record.parent, None);
}
}