use std::path::Path;
use std::sync::Arc;
use pulpo_common::event::PulpoEvent;
use pulpo_common::session::{InterventionCode, Session, SessionStatus, status_reason};
use tracing::{debug, info};
use super::{
HarnessSignals, IdleAction, IdleConfig, ReadyContext, build_session_event,
detect_and_store_output_metadata, detect_waiting_for_input, owned_signals, resolve_backend_id,
};
use crate::backend::Backend;
use crate::store::Store;
pub(super) async fn check_idle_sessions(
backend: &Arc<dyn Backend>,
store: &Store,
idle_config: &IdleConfig,
ready_ctx: &ReadyContext,
extra_waiting_patterns: &[String],
) {
let sessions = super::list_sessions_or_warn(store, "Idle check").await;
let live: Vec<_> = sessions
.iter()
.filter(|session| {
session.status == SessionStatus::Working || session.status == SessionStatus::Waiting
})
.collect();
let now = chrono::Utc::now();
let timeout =
chrono::Duration::seconds(idle_config.timeout_secs.try_into().unwrap_or(i64::MAX));
for session in live {
check_session_idle(
backend,
store,
idle_config,
session,
now,
timeout,
ready_ctx,
extra_waiting_patterns,
)
.await;
}
}
pub(super) fn effective_idle_threshold_secs(
session: &Session,
idle_config: &IdleConfig,
) -> Option<u64> {
match session.idle_threshold_secs {
Some(0) => None,
Some(secs) => Some(u64::from(secs)),
None => Some(idle_config.threshold_secs),
}
}
pub(super) async fn check_session_idle(
backend: &Arc<dyn Backend>,
store: &Store,
idle_config: &IdleConfig,
session: &Session,
now: chrono::DateTime<chrono::Utc>,
timeout: chrono::Duration,
ready_ctx: &ReadyContext,
extra_waiting_patterns: &[String],
) {
let backend_id = resolve_backend_id(session, backend.as_ref());
match backend.is_alive(&backend_id) {
Ok(true) => {}
Ok(false) => {
resolve_and_report_dead_session(store, backend, &backend_id, session, ready_ctx).await;
return;
}
#[allow(unused_variables)]
Err(error) => {
debug!(
"Idle check: failed to check liveness for {}: {error}",
session.name
);
return;
}
}
let current_output = match backend.capture_output(&backend_id, 500) {
Ok(output) => output,
#[allow(unused_variables)]
Err(error) => {
debug!(
"Idle check: failed to capture output for {}: {error}",
session.name
);
return;
}
};
#[allow(unused_variables)]
if let Err(error) = store
.update_session_output_snapshot(&session.id.to_string(), ¤t_output)
.await
{
coverage_warn!(
"Idle check: failed to update output snapshot for {}: {error}",
session.name
);
return;
}
let signals = owned_signals(session);
let exact_usage =
crate::usage::read_exact_usage_for_session(session, Path::new(store.data_dir()));
detect_and_store_output_metadata(store, session, ¤t_output, exact_usage, signals).await;
let output_changed = session.output_snapshot.as_deref() != Some(current_output.as_str());
if output_changed {
handle_active_session(store, session, ready_ctx, signals).await;
return;
}
if !signals.lifecycle && session.status == SessionStatus::Working {
let immediate = detect_waiting_for_input(¤t_output, extra_waiting_patterns);
let effective_threshold_secs = effective_idle_threshold_secs(session, idle_config);
let last_change = session.last_output_at.unwrap_or(session.created_at);
let sustained = effective_threshold_secs.is_some_and(|threshold_secs| {
(now - last_change).num_seconds() >= i64::try_from(threshold_secs).unwrap_or(i64::MAX)
});
if immediate || sustained {
let reason = if immediate {
status_reason::needs_input("permission")
} else {
status_reason::IDLE.to_owned()
};
info!(
"Session {} idle ({}), transitioning to waiting",
session.name,
if immediate {
"waiting pattern"
} else {
"output unchanged"
}
);
#[allow(unused_variables)]
if let Err(error) = store
.update_session_status(
&session.id.to_string(),
SessionStatus::Waiting,
Some(&reason),
)
.await
{
coverage_warn!(
"Idle check: failed to transition {} to waiting: {error}",
session.name
);
} else if let Some(tx) = &ready_ctx.event_tx {
let event = build_session_event(
session,
SessionStatus::Waiting,
Some(&reason),
Some(SessionStatus::Working),
&ready_ctx.node_name,
Some(current_output.clone()),
);
let _ = tx.send(PulpoEvent::Session(event));
}
return;
}
}
let blocked_on_operator = session.status == SessionStatus::Waiting
&& session
.status_reason
.as_deref()
.and_then(status_reason::needs_input_reason)
.is_some();
if blocked_on_operator {
return;
}
handle_idle_session(
backend,
store,
idle_config,
session,
now,
timeout,
ready_ctx,
)
.await;
}
#[cfg_attr(coverage, allow(unused_variables))]
async fn resolve_and_report_dead_session(
store: &Store,
backend: &Arc<dyn Backend>,
backend_id: &str,
session: &Session,
ready_ctx: &ReadyContext,
) {
let previous = session.status;
let mut updated = session.clone();
if let Err(error) = crate::session::manager::resolve_dead_backend_session(
store,
backend.as_ref(),
backend_id,
&mut updated,
)
.await
{
coverage_warn!(
"Idle check: failed to resolve dead backend for {}: {error}",
session.name
);
return;
}
info!(
"Session {} backend is gone, resolved {previous} -> {}",
session.name, updated.status
);
if let Some(tx) = &ready_ctx.event_tx {
let event = build_session_event(
&updated,
updated.status,
updated.status_reason.as_deref(),
Some(previous),
&ready_ctx.node_name,
updated.output_snapshot.clone(),
);
let _ = tx.send(PulpoEvent::Session(event));
}
}
pub(super) async fn handle_active_session(
store: &Store,
session: &Session,
ready_ctx: &ReadyContext,
signals: HarnessSignals,
) {
if signals.lifecycle {
return;
}
if session.status == SessionStatus::Waiting {
info!(
"Session {} has new output, transitioning back to active",
session.name
);
#[allow(unused_variables)]
if let Err(error) = store
.update_session_status(&session.id.to_string(), SessionStatus::Working, None)
.await
{
coverage_warn!(
"Idle check: failed to transition {} back to active: {error}",
session.name
);
} else if let Some(tx) = &ready_ctx.event_tx {
let event = build_session_event(
session,
SessionStatus::Working,
None,
Some(SessionStatus::Waiting),
&ready_ctx.node_name,
session.output_snapshot.clone(),
);
let _ = tx.send(PulpoEvent::Session(event));
}
}
if session.idle_since.is_none() {
return;
}
info!(
"Idle check: session {} active again, clearing idle status",
session.name
);
#[allow(unused_variables)]
if let Err(error) = store
.clear_session_idle_since(&session.id.to_string())
.await
{
coverage_warn!(
"Idle check: failed to clear idle_since for {}: {error}",
session.name
);
}
}
pub(super) async fn handle_idle_session(
backend: &Arc<dyn Backend>,
store: &Store,
idle_config: &IdleConfig,
session: &Session,
now: chrono::DateTime<chrono::Utc>,
timeout: chrono::Duration,
ready_ctx: &ReadyContext,
) {
let last_activity = session.last_output_at.unwrap_or(session.created_at);
let idle_duration = now - last_activity;
if idle_duration <= timeout {
return;
}
let minutes = idle_duration.num_minutes();
match idle_config.action {
IdleAction::Alert => {
if session.idle_since.is_none() {
coverage_warn!(
"Idle check: session {} idle for {minutes} minutes, marking as idle",
session.name
);
#[allow(unused_variables)]
if let Err(error) = store
.update_session_idle_since(&session.id.to_string())
.await
{
coverage_warn!(
"Idle check: failed to set idle_since for {}: {error}",
session.name
);
}
}
}
IdleAction::Kill => {
let reason = format!("Idle for {minutes} minutes");
if !super::intervention::stop_and_record(
backend,
store,
session,
InterventionCode::IdleTimeout,
&reason,
ready_ctx,
"Idle check: failed to kill idle session",
"Idle check: failed to record intervention",
)
.await
{
return;
}
coverage_warn!(
"Idle check: stopped idle session {} after {minutes} minutes",
session.name
);
}
}
}