use std::sync::{Arc, Weak};
use std::time::Duration;
use turul_a2a_types::{Message, Part, Role, TaskState, TaskStatus};
use crate::router::{AppState, commit_framework_terminal};
use crate::server::in_flight::InFlightRegistry;
use crate::storage::{A2aCancellationSupervisor, A2aStorageError, SupervisedTask};
const LEASE_LOST_REASON: &str =
"task failed: the server running its executor stopped renewing the executor's lease";
pub(crate) fn spawn_loops(state: &AppState) {
let config = &state.runtime_config;
if config.executor_lease_ttl.is_none()
&& config.executor_stall_timeout.is_none()
&& config.idle_task_timeout.is_none()
{
return;
}
let registry = Arc::downgrade(&state.in_flight);
let supervisor = Arc::clone(&state.cancellation_supervisor);
let committer = AppState {
in_flight: Arc::new(InFlightRegistry::new()),
..state.clone()
};
if let Some(ttl) = config.executor_lease_ttl {
tokio::spawn(renew_loop(registry.clone(), supervisor.clone(), ttl));
tokio::spawn(reap_loop(
registry.clone(),
supervisor,
committer.clone(),
ttl,
));
}
if let Some(timeout) = config.idle_task_timeout {
tokio::spawn(idle_loop(registry.clone(), committer.clone(), timeout));
}
if let Some(timeout) = config.executor_stall_timeout {
tokio::spawn(stall_loop(
registry,
committer,
timeout,
config.timeout_abort_grace,
));
}
}
fn failed_status(reason: &str) -> TaskStatus {
let message = Message::new(
uuid::Uuid::now_v7().to_string(),
Role::Agent,
vec![Part::text(reason)],
);
TaskStatus::new(TaskState::Failed).with_message(message)
}
fn now_micros() -> i64 {
chrono::Utc::now().timestamp_micros()
}
async fn renew_loop(
registry: Weak<InFlightRegistry>,
supervisor: Arc<dyn A2aCancellationSupervisor>,
ttl: Duration,
) {
let period = ttl / 3;
let mut ticker = tokio::time::interval_at(tokio::time::Instant::now() + period, period);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
ticker.tick().await;
let Some(groups) = registry.upgrade().map(|r| r.snapshot_by_tenant()) else {
return;
};
let expires_at =
now_micros().saturating_add(i64::try_from(ttl.as_micros()).unwrap_or(i64::MAX));
let supervisor = supervisor.as_ref();
futures::future::join_all(groups.iter().map(|(tenant, task_ids)| async move {
if let Err(e) = supervisor
.supervisor_renew_executor_leases(tenant, task_ids, Some(expires_at))
.await
{
tracing::warn!(
target: "turul_a2a::executor_lease_renew_error",
%tenant,
error = %e,
"executor lease renewal failed; will retry next tick"
);
}
}))
.await;
}
}
async fn reap_loop(
registry: Weak<InFlightRegistry>,
supervisor: Arc<dyn A2aCancellationSupervisor>,
committer: AppState,
ttl: Duration,
) {
loop {
if registry.strong_count() == 0 {
return;
}
reap_once(supervisor.as_ref(), &committer).await;
tokio::time::sleep(ttl).await;
}
}
async fn reap_once(supervisor: &dyn A2aCancellationSupervisor, committer: &AppState) {
let expired = match supervisor
.supervisor_list_expired_executor_leases(now_micros())
.await
{
Ok(expired) => expired,
Err(e) => {
tracing::warn!(
target: "turul_a2a::executor_lease_reap_error",
error = %e,
"listing expired executor leases failed; will retry next tick"
);
return;
}
};
for task in expired {
match commit_framework_terminal(
committer,
&task.tenant,
&task.owner,
&task.task_id,
&task.context_id,
failed_status(LEASE_LOST_REASON),
)
.await
{
Ok(_) => {
tracing::warn!(
target: "turul_a2a::executor_lease_expired",
tenant = %task.tenant,
task_id = %task.task_id,
"failed task whose executor lease expired"
);
clear_lease(supervisor, &task.tenant, &task.task_id).await;
}
Err(A2aStorageError::TerminalStateAlreadySet { .. })
| Err(A2aStorageError::TaskNotFound(_)) => {
clear_lease(supervisor, &task.tenant, &task.task_id).await;
}
Err(e) => tracing::warn!(
target: "turul_a2a::executor_lease_reap_error",
tenant = %task.tenant,
task_id = %task.task_id,
error = %e,
"failing task with expired executor lease failed; will retry next tick"
),
}
}
}
async fn clear_lease(supervisor: &dyn A2aCancellationSupervisor, tenant: &str, task_id: &str) {
if let Err(e) = supervisor
.supervisor_renew_executor_leases(tenant, &[task_id.to_string()], None)
.await
{
tracing::warn!(
target: "turul_a2a::executor_lease_renew_error",
%tenant,
%task_id,
error = %e,
"clearing executor lease failed"
);
}
}
async fn stall_loop(
registry: Weak<InFlightRegistry>,
committer: AppState,
timeout: Duration,
abort_grace: Duration,
) {
loop {
tokio::time::sleep(timeout / 4).await;
let Some(handles) = registry.upgrade().map(|r| r.snapshot_handles()) else {
return;
};
for ((tenant, task_id), handle) in handles {
if handle.terminal_committed()
|| handle.client_held()
|| handle.cancellation.is_cancelled()
|| handle.since_progress() <= timeout
{
continue;
}
let Some((owner, context_id)) = handle.identity() else {
continue;
};
let reason = format!("task failed: its executor made no progress for {timeout:?}");
match commit_framework_terminal(
&committer,
&tenant,
owner,
&task_id,
context_id,
failed_status(&reason),
)
.await
{
Ok(_) => tracing::warn!(
target: "turul_a2a::executor_stalled",
%tenant,
%task_id,
"failed task whose executor made no progress"
),
Err(
A2aStorageError::TerminalStateAlreadySet { .. }
| A2aStorageError::TaskNotFound(_),
) => {}
Err(e) => {
tracing::warn!(
target: "turul_a2a::executor_stalled",
%tenant,
%task_id,
error = %e,
"failing stalled task failed; will retry next tick"
);
continue;
}
}
handle.cancel_then_abort_after(abort_grace);
}
}
}
const IDLE_PAGE: usize = 100;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct IdleSweep {
pub canceled: usize,
pub errors: usize,
}
async fn idle_loop(registry: Weak<InFlightRegistry>, committer: AppState, timeout: Duration) {
loop {
let Some(live) = registry.upgrade() else {
return;
};
let _ = cancel_idle_tasks(&committer, Some(&live), timeout).await;
drop(live);
tokio::time::sleep(timeout / 4).await;
}
}
#[doc(hidden)]
pub async fn cancel_idle_tasks(
state: &AppState,
registry: Option<&InFlightRegistry>,
idle_for: Duration,
) -> Result<IdleSweep, A2aStorageError> {
let mut sweep = IdleSweep::default();
let reason = format!("task canceled: no client input for {idle_for:?}");
loop {
let before =
now_micros().saturating_sub(i64::try_from(idle_for.as_micros()).unwrap_or(i64::MAX));
let page = state
.cancellation_supervisor
.supervisor_list_idle_client_held_tasks(before, IDLE_PAGE)
.await
.inspect_err(|e| {
tracing::warn!(
target: "turul_a2a::idle_task_sweep_error",
error = %e,
"listing idle tasks failed; will retry next sweep"
)
})?;
let mut page_errors = 0;
for task in &page {
match cancel_idle_task(state, registry, task, &reason).await {
IdleOutcome::Canceled => sweep.canceled += 1,
IdleOutcome::Resolved => {}
IdleOutcome::Failed => page_errors += 1,
}
}
sweep.errors += page_errors;
if page.len() < IDLE_PAGE || page_errors == page.len() {
return Ok(sweep);
}
}
}
enum IdleOutcome {
Canceled,
Resolved,
Failed,
}
async fn cancel_idle_task(
state: &AppState,
registry: Option<&InFlightRegistry>,
task: &SupervisedTask,
reason: &str,
) -> IdleOutcome {
let message = Message::new(
uuid::Uuid::now_v7().to_string(),
Role::Agent,
vec![Part::text(reason)],
);
let status = TaskStatus::new(TaskState::Canceled).with_message(message);
match commit_framework_terminal(
state,
&task.tenant,
&task.owner,
&task.task_id,
&task.context_id,
status,
)
.await
{
Ok(_) => {
tracing::warn!(
target: "turul_a2a::idle_task_canceled",
tenant = %task.tenant,
task_id = %task.task_id,
"canceled task left waiting on its client"
);
if let Some(handle) =
registry.and_then(|r| r.get(&(task.tenant.clone(), task.task_id.clone())))
{
handle.cancel_then_abort_after(state.runtime_config.cancel_abort_grace);
}
IdleOutcome::Canceled
}
Err(A2aStorageError::TerminalStateAlreadySet { .. } | A2aStorageError::TaskNotFound(_)) => {
IdleOutcome::Resolved
}
Err(e) => {
tracing::warn!(
target: "turul_a2a::idle_task_sweep_error",
tenant = %task.tenant,
task_id = %task.task_id,
error = %e,
"canceling idle task failed; will retry next sweep"
);
IdleOutcome::Failed
}
}
}