use actl_core::state::{SignalPaths, unix_ms};
const PROBE_INTERVAL_MS: u64 = 5000;
const PROBE_GIVE_UP_MS: u64 = 10 * 60 * 1000;
const AUTO_RESUME_BUDGET: u32 = 2;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Verdict {
Retriable,
AlreadyDone,
Pending,
}
pub fn verdict(before_ok: Option<bool>, verify_ok: Option<bool>) -> Verdict {
match (before_ok, verify_ok) {
(_, Some(true)) => Verdict::AlreadyDone,
(Some(true), _) => Verdict::Retriable,
_ => Verdict::Pending,
}
}
fn newest_probe_target(paths: &SignalPaths) -> Option<String> {
let now = unix_ms();
let dir = std::fs::read_dir(paths.dir.join("flows")).ok()?;
let mut best: Option<(u64, String)> = None;
for entry in dir.flatten() {
let id = entry.file_name().into_string().ok()?;
let Some(state) = super::flow_ui::read(paths, &id) else {
continue;
};
if state["status"] != "paused" || state["reason"] != "action_failed" {
continue;
}
let updated = state["updated_ms"].as_u64()?;
if now.saturating_sub(updated) < PROBE_GIVE_UP_MS
&& best.as_ref().is_none_or(|(t, _)| updated > *t)
{
best = Some((updated, id));
}
}
best.map(|(_, id)| id)
}
pub fn tick(
paths: &SignalPaths,
bound: Option<&str>,
status: Option<&str>,
reason: Option<&str>,
paused_since_ms: u64,
plan: Option<&serde_json::Value>,
) -> Option<(String, Verdict)> {
let _trace = actl_core::trace::scope("signal.repair_probe");
#[derive(Clone, PartialEq, Eq)]
struct Key {
root: std::path::PathBuf,
bound: Option<String>,
status: Option<String>,
reason: Option<String>,
updated: u64,
generation: String,
}
#[derive(Clone)]
struct Result {
id: String,
verdict: Verdict,
updated: u64,
}
thread_local! {
static WORKER: std::cell::RefCell<super::probe_worker::Worker<Key, Option<Result>>> = std::cell::RefCell::new(Default::default());
}
let generation = paths.execution_generation().ok()?;
if paths.stop_requested() {
return None;
}
let key = Key {
root: paths.dir.clone(),
bound: bound.map(str::to_string),
status: status.map(str::to_string),
reason: reason.map(str::to_string),
updated: paused_since_ms,
generation,
};
let input = key.clone();
let worker_paths = paths.clone();
let plan = plan.cloned();
let result = WORKER.with(|worker| {
worker.borrow_mut().poll(
key,
std::time::Duration::from_millis(PROBE_INTERVAL_MS),
move || {
let (id, verdict, updated) = evaluate(
&worker_paths,
input.bound.as_deref(),
input.status.as_deref(),
input.reason.as_deref(),
input.updated,
plan.as_ref(),
)?;
Some(Result {
id,
verdict,
updated,
})
},
)
});
let result = match result {
Ok(result) => result??,
Err(error) => {
eprintln!("INTERNAL: repair probe unavailable: {error}");
return None;
}
};
let state = super::flow_ui::read(paths, &result.id)?;
if revision(&state) != Some(result.updated) {
return None;
}
Some((result.id, result.verdict))
}
fn revision(state: &serde_json::Value) -> Option<u64> {
if state["status"] != "paused" || state["reason"] != "action_failed" {
return None;
}
state["updated_ms"]
.as_u64()
.filter(|at| unix_ms().saturating_sub(*at) < PROBE_GIVE_UP_MS)
}
fn evaluate(
paths: &SignalPaths,
bound: Option<&str>,
status: Option<&str>,
reason: Option<&str>,
paused_since_ms: u64,
plan: Option<&serde_json::Value>,
) -> Option<(String, Verdict, u64)> {
let eligible = bound.is_some()
&& status == Some("paused")
&& reason == Some("action_failed")
&& plan
.map(|p| p["steps"].as_array().is_some_and(|s| !s.is_empty()))
.unwrap_or(false)
&& unix_ms().saturating_sub(paused_since_ms) < PROBE_GIVE_UP_MS;
let id = bound
.filter(|_| eligible)
.map(str::to_string)
.or_else(|| newest_probe_target(paths))?;
let id = id.as_str();
let plan_value = match plan {
Some(p) if bound == Some(id) => p.clone(),
_ => super::ui_text::plan(paths, id)?,
};
let flow = super::flow_ui::read(paths, id)?;
let updated = revision(&flow)?;
let steps = flow["steps"].as_array()?;
let plan_steps = plan_value["steps"].as_array()?;
let index = steps
.iter()
.position(|s| s["status"] != "completed")
.or_else(|| steps.len().checked_sub(1))?;
let run_probe = |invocation: Option<&serde_json::Value>| -> Option<bool> {
let invocation = invocation?;
let mut args = vec![invocation["command"].as_str()?.to_string()];
if let Some(list) = invocation["args"].as_array() {
args.extend(list.iter().filter_map(|a| a.as_str().map(str::to_string)));
}
run_actl(args).ok()
};
let before_ok = run_probe(plan_steps.get(index).and_then(|s| s.get("before")));
let verify_ok = run_probe(plan_steps.get(index).and_then(|s| s.get("verify")));
let current = super::flow_ui::read(paths, id)?;
if revision(¤t) != Some(updated) {
return None;
}
let verdict = verdict(before_ok, verify_ok);
record(
paths,
id,
"repair_probe",
&serde_json::json!({
"step_index": index,
"before_ok": before_ok,
"verify_ok": verify_ok,
"verdict": format!("{verdict:?}").to_lowercase(),
}),
);
Some((id.to_string(), verdict, updated))
}
pub fn record(paths: &SignalPaths, id: &str, event: &str, data: &serde_json::Value) {
use std::io::Write;
let line = serde_json::json!({
"event": event, "ts_ms": unix_ms(), "data": data,
});
let Ok(mut file) = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(paths.dir.join("flows").join(id).join("probe.jsonl"))
else {
return;
};
let _ = writeln!(file, "{line}");
}
pub fn consume_auto_resume(paths: &SignalPaths, id: &str, declared: bool, v: Verdict) -> bool {
if !declared || v != Verdict::Retriable {
return false;
}
let used = std::fs::read_to_string(paths.dir.join("flows").join(id).join("probe.jsonl"))
.map(|text| {
text.lines()
.filter(|l| l.contains("\"event\":\"auto_resume\""))
.count()
})
.unwrap_or(0);
(used as u32) < AUTO_RESUME_BUDGET
}
fn run_actl(args: Vec<String>) -> Result<bool, ()> {
use std::os::windows::process::CommandExt;
let exe = std::env::current_exe()
.map_err(|_| ())?
.with_file_name("actl.exe");
let output = std::process::Command::new(exe)
.args(&args)
.env("ACTL_SILENT_PROBE", "1")
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null())
.creation_flags(0x08000000)
.output()
.map_err(|_| ())?;
let body: serde_json::Value = serde_json::from_slice(&output.stdout).map_err(|_| ())?;
Ok(output.status.success() && body["ok"] == true)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_current_action_failure_can_use_a_worker_result() {
let now = unix_ms();
let mut state =
serde_json::json!({"status":"paused","reason":"action_failed","updated_ms":now});
assert_eq!(revision(&state), Some(now));
for status in ["completed", "running", "stopped", "needs_review"] {
state["status"] = serde_json::json!(status);
assert_eq!(revision(&state), None);
}
state["status"] = serde_json::json!("paused");
state["reason"] = serde_json::json!("pause_requested");
assert_eq!(revision(&state), None);
state["reason"] = serde_json::json!("action_failed");
state["updated_ms"] = serde_json::json!(now - PROBE_GIVE_UP_MS);
assert_eq!(revision(&state), None);
}
#[test]
#[ignore = "read-only live discovery timing; set ACTL_PROBE_PROFILE_OUTPUT"]
fn profile_discovery() -> Result<(), Box<dyn std::error::Error>> {
let file = std::fs::File::create_new(std::env::var("ACTL_PROBE_PROFILE_OUTPUT")?)?;
let paths = SignalPaths::default();
let start = std::time::Instant::now();
let target = newest_probe_target(&paths);
serde_json::to_writer_pretty(
file,
&serde_json::json!({
"duration_ms":start.elapsed().as_millis(),"target":target,
"binary":std::env::current_exe()?,"ts_ms":unix_ms(),
"flow_count":std::fs::read_dir(paths.dir.join("flows"))?.count(),
"input_injected":false,"flow_execution":false
}),
)?;
Ok(())
}
#[test]
fn verdict_prioritizes_already_done_over_retriable() {
assert_eq!(verdict(Some(true), Some(true)), Verdict::AlreadyDone);
assert_eq!(verdict(Some(false), Some(true)), Verdict::AlreadyDone);
assert_eq!(verdict(Some(true), None), Verdict::Retriable);
assert_eq!(verdict(Some(true), Some(false)), Verdict::Retriable);
assert_eq!(verdict(None, None), Verdict::Pending);
assert_eq!(verdict(Some(false), Some(false)), Verdict::Pending);
}
}