use std::path::Path;
use std::time::{Duration, Instant};
use serde::Serialize;
use octl_core::{read_manifest_opt, read_node_opt, Node, NodeId, RunLock, Status};
use crate::error::CliError;
use crate::output::{self, OutputFormat, OutputSpec};
use crate::run::merge::{self, ConsumerOutcome};
use crate::run::{from_core, run_paths_from_cli_arg};
use crate::supervise::{pid_file, watchdog};
const DEFAULT_NODE_ID: &str = "n-0001";
const FENCE_GRACE: Duration = Duration::from_secs(5);
const FENCE_POLL: Duration = Duration::from_millis(100);
pub struct Args<'a> {
pub run_id: String,
pub source: Option<String>,
pub report_file: Option<std::path::PathBuf>,
pub fence: bool,
pub dry_run: bool,
pub spec: &'a OutputSpec,
pub warnings: &'a [String],
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum WorkerState {
Exited,
NoPid,
Gone,
Live { pid: u32, start_time: u64 },
Unverifiable { pid: u32 },
}
impl WorkerState {
fn wire(self) -> &'static str {
match self {
WorkerState::Exited => "exited",
WorkerState::NoPid => "no-pid",
WorkerState::Gone => "gone",
WorkerState::Live { .. } => "live",
WorkerState::Unverifiable { .. } => "unverifiable",
}
}
}
fn classify_worker(node: &Node) -> WorkerState {
if let Some(live) = positive_live_identity(node) {
return live;
}
if node.worker_exit.is_some() {
return WorkerState::Exited;
}
let Some(pid_i) = node.agent_pid else {
return WorkerState::NoPid;
};
if pid_i <= 0 {
return WorkerState::NoPid;
}
let pid = pid_i as u32;
if !pid_file::pid_alive(pid) {
return WorkerState::Gone;
}
match node
.agent_pid_start_time
.map(|t| t.timestamp().max(0) as u64)
{
Some(_) => WorkerState::Gone,
None => WorkerState::Unverifiable { pid },
}
}
fn positive_live_identity(node: &Node) -> Option<WorkerState> {
let pid_i = node.agent_pid?;
if pid_i <= 0 {
return None;
}
let pid = pid_i as u32;
if !pid_file::pid_alive(pid) {
return None;
}
let expected = node
.agent_pid_start_time
.map(|t| t.timestamp().max(0) as u64)?;
let actual = watchdog::pid_start_time(pid)?;
(expected.abs_diff(actual) <= 1).then_some(WorkerState::Live {
pid,
start_time: expected,
})
}
fn fence_worker(pid: u32, expected_start: u64) -> Result<(), CliError> {
let Some(pid_t) = pid_file::to_pid_t(pid) else {
return Err(CliError::system(
"fence_failed",
format!("worker pid {pid} is out of range; refusing to signal"),
));
};
match watchdog::pid_start_time(pid) {
Some(actual) if expected_start.abs_diff(actual) <= 1 => {}
_ => return Ok(()),
}
let rc = unsafe { libc::kill(pid_t, libc::SIGTERM) };
if rc != 0 {
let err = std::io::Error::last_os_error();
if err.raw_os_error() == Some(libc::ESRCH) {
return Ok(());
}
return Err(CliError::system(
"fence_failed",
format!("SIGTERM to worker pid {pid} failed: {err}"),
));
}
let deadline = Instant::now() + FENCE_GRACE;
while Instant::now() < deadline {
if !pid_file::pid_alive(pid) {
return Ok(());
}
std::thread::sleep(FENCE_POLL);
}
if pid_file::pid_alive(pid) {
return Err(CliError::user(
"fence_failed",
format!(
"worker pid {pid} did not exit within {}s of SIGTERM — refusing to \
merge over a live worker. Investigate the process (it may be blocked \
in an uninterruptible state) before retrying.",
FENCE_GRACE.as_secs()
),
));
}
Ok(())
}
#[derive(Serialize)]
struct MergeSummary {
branch: String,
#[serde(skip_serializing_if = "Option::is_none")]
source: Option<String>,
merged: bool,
#[serde(skip_serializing_if = "Option::is_none")]
report_seq: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
supervisor: Option<ConsumerOutcome>,
}
#[derive(Serialize)]
struct SalvagePayload {
run_id: String,
node_id: String,
worker_state: &'static str,
fenced: bool,
merge: MergeSummary,
#[serde(skip_serializing_if = "std::ops::Not::not")]
dry_run: bool,
}
pub fn run(args: Args<'_>) -> Result<(), CliError> {
let root = crate::home::root_dir()?;
let paths = run_paths_from_cli_arg(&root, &args.run_id)?;
let run_id = paths.run_id.as_str().to_string();
let node_id = NodeId::parse_str(DEFAULT_NODE_ID).expect("DEFAULT_NODE_ID is valid");
let (manifest, node) = RunLock::with_shared_lock(&paths.lock(), || {
let manifest = read_manifest_opt(&paths)?;
let node = read_node_opt(&paths, &node_id)?;
Ok((manifest, node))
})
.map_err(from_core)?;
let manifest = manifest.ok_or_else(|| {
CliError::user("run_not_found", format!("no run with id {run_id}"))
.with_invalid_value(&run_id)
})?;
crate::run::reject_legacy_kind(manifest.kind, &run_id)?;
match manifest.status {
Status::Done => {
let worktree_present = node
.as_ref()
.and_then(|n| n.worktree_path.as_deref())
.is_some_and(|p| Path::new(p).try_exists().unwrap_or(false));
let hint = if worktree_present {
format!(
" but its worktree still exists — teardown did not finish. Run \
`orchestratectl run reattach {run_id}` to complete it (a re-merge is not \
needed; the work already landed)."
)
} else {
" — its work merged and its worktree was torn down; there is nothing to salvage"
.to_string()
};
return Err(CliError::user(
"run_already_terminal",
format!("run {run_id} is already done{hint}"),
)
.with_invalid_value(&run_id));
}
Status::Cancelled => {
return Err(CliError::user(
"run_already_terminal",
format!(
"run {run_id} was cancelled — a cancelled run never adopts a merge, so \
salvage cannot finish it"
),
)
.with_invalid_value(&run_id));
}
Status::Pending | Status::Running | Status::Blocked | Status::Failed => {}
}
if manifest.node_count != 1 {
return Err(CliError::user(
"ambiguous_multi_node",
format!(
"run {run_id} has {} nodes — salvage targets a single-worker run only. \
Per-node salvage of a fan-out is not yet supported.",
manifest.node_count
),
)
.with_invalid_value(&run_id)
.with_expected(serde_json::json!({ "node_count": 1 })));
}
let node = node.ok_or_else(|| {
CliError::user(
"node_not_found",
format!("run {run_id} has no {node_id} node to salvage"),
)
.with_invalid_value(node_id.as_str())
})?;
let worktree_path = node.worktree_path.as_deref().ok_or_else(|| {
CliError::user(
"no_worktree",
format!(
"node {node_id} has no preserved worktree — nothing to salvage (a driver \
node, or the worktree was already removed)"
),
)
.with_invalid_value(node_id.as_str())
})?;
let has_branch = node.branch.as_deref().is_some_and(|s| !s.is_empty());
if !has_branch {
return Err(CliError::user(
"no_branch",
format!("node {node_id} has no preserved branch recorded; cannot salvage"),
)
.with_invalid_value(node_id.as_str()));
}
if !Path::new(worktree_path).try_exists().unwrap_or(true) {
return Err(CliError::user(
"worktree_missing",
format!(
"worktree {worktree_path} no longer exists — its work was likely already \
merged or torn down; there is nothing to salvage"
),
)
.with_invalid_value(&run_id));
}
let worker = classify_worker(&node);
if manifest.status == Status::Pending && worker == WorkerState::NoPid {
return Err(CliError::user(
"run_not_started",
format!(
"run {run_id} is still pending and its worker never started (no recorded \
agent pid, no worker exit) — there is no work to salvage. Cancel it with \
`orchestratectl run cancel {run_id}` if it is stuck."
),
)
.with_invalid_value(&run_id));
}
let needs_fence = match worker {
WorkerState::Exited | WorkerState::NoPid | WorkerState::Gone => false,
WorkerState::Unverifiable { pid } => {
return Err(CliError::user(
"worker_unfenceable",
format!(
"run {run_id}'s worker pid {pid} is alive but its identity cannot be \
verified (no recorded start-time) — refusing to signal a process that \
may have been recycled. Confirm the worker is gone, then retry."
),
)
.with_invalid_value(&run_id));
}
WorkerState::Live { pid, .. } => {
if !args.fence {
return Err(CliError::user(
"worker_live",
format!(
"run {run_id}'s original worker (pid {pid}) is still alive. Salvage \
will not kill a running worker implicitly — re-run with `--fence` to \
SIGTERM it and finish the run, or let it complete on its own."
),
)
.with_invalid_value(&run_id));
}
true
}
};
let preview = merge::execute(&merge::Args {
run_id: run_id.clone(),
source: args.source.clone(),
node_id: None,
report_file: args.report_file.clone(),
dry_run: true,
spec: args.spec,
warnings: args.warnings,
})?;
if args.dry_run {
let mut warnings = preview.warnings.clone();
if let WorkerState::Live { pid, .. } = worker {
warnings.push(format!(
"would fence worker pid {pid} (SIGTERM) before finishing the run"
));
}
return emit(
&SalvagePayload {
run_id,
node_id: node_id.as_str().to_string(),
worker_state: worker.wire(),
fenced: needs_fence,
merge: merge_summary(&preview),
dry_run: true,
},
args.spec,
&warnings,
);
}
let mut base_warnings: Vec<String> = args.warnings.to_vec();
let fenced = if needs_fence {
if let WorkerState::Live { pid, start_time } = worker {
fence_worker(pid, start_time)?;
base_warnings.push(format!(
"fenced worker pid {pid} (SIGTERM) before finishing the run"
));
}
true
} else {
false
};
let mo = merge::execute(&merge::Args {
run_id: run_id.clone(),
source: args.source.clone(),
node_id: None,
report_file: args.report_file.clone(),
dry_run: false,
spec: args.spec,
warnings: &base_warnings,
})
.map_err(|mut e| {
if fenced {
e.message = format!(
"{} (NOTE: the prior worker was already fenced/SIGTERM'd; the worktree and \
branch are preserved — resolve the merge and re-run `run salvage`/`run merge`)",
e.message
);
}
e
})?;
let out_warnings = mo.warnings.clone();
emit(
&SalvagePayload {
run_id,
node_id: node_id.as_str().to_string(),
worker_state: worker.wire(),
fenced,
merge: merge_summary(&mo),
dry_run: false,
},
args.spec,
&out_warnings,
)
}
fn merge_summary(mo: &merge::MergeOutcome) -> MergeSummary {
MergeSummary {
branch: mo.branch.clone(),
source: mo.source.clone(),
merged: mo.merged,
report_seq: mo.report_seq,
supervisor: mo.supervisor.clone(),
}
}
fn emit(payload: &SalvagePayload, spec: &OutputSpec, warnings: &[String]) -> Result<(), CliError> {
match spec.format {
OutputFormat::Json | OutputFormat::Jsonl => {
output::emit_envelope(payload, spec, warnings)?;
}
OutputFormat::Text => {
println!("run-id: {}", payload.run_id);
println!("node-id: {}", payload.node_id);
println!("worker-state: {}", payload.worker_state);
println!("fenced: {}", payload.fenced);
println!("branch: {}", payload.merge.branch);
match &payload.merge.source {
Some(s) => println!("source: {s}"),
None => println!("source: (auto-detect main/master)"),
}
if payload.dry_run {
println!("note: --dry-run (no fence, no merge)");
} else {
println!("merged: {}", payload.merge.merged);
if let Some(seq) = payload.merge.report_seq {
println!("report_seq: {seq}");
}
}
output::emit_text_warnings(warnings);
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use std::process::Command;
use chrono::{DateTime, Utc};
use octl_core::{Kind, RunId, WorkerExit};
fn node(
agent_pid: Option<i32>,
start_time: Option<DateTime<Utc>>,
exit: Option<WorkerExit>,
) -> Node {
Node {
schema_version: 1,
node_id: NodeId::parse_str("n-0001").unwrap(),
run_id: RunId::parse_str("01jxsnap000000000000000000").unwrap(),
parent_node_id: None,
kind: Kind::Spinoff,
status: Status::Running,
task: None,
worktree_path: Some("/tmp/wt".into()),
branch: Some("wt/x".into()),
base_sha: None,
tmux_window: None,
tmux_identity: None,
agent_pid,
agent_pid_start_time: start_time,
supervisor_pid: None,
children: vec![],
started_at: None,
updated_at: Utc::now(),
last_report: None,
last_processed_report_seq_by_child: serde_json::Map::new(),
retry_attempts: 0,
worker_exit: exit,
pending_merge: None,
first_death_at: None,
}
}
fn spawn_sleeper() -> std::process::Child {
Command::new("sleep")
.arg("30")
.spawn()
.expect("spawn sleep")
}
#[test]
fn told_exit_wins_when_live_pid_identity_unprovable() {
let mut child = spawn_sleeper();
let exit = WorkerExit {
code: Some(0),
signal: None,
at: Utc::now(),
};
let n = node(Some(child.id() as i32), None, Some(exit));
assert_eq!(classify_worker(&n), WorkerState::Exited);
let _ = child.kill();
let _ = child.wait();
}
#[test]
fn live_matching_identity_overrides_a_stale_told_exit() {
let mut child = spawn_sleeper();
let pid = child.id();
let st = watchdog::pid_start_time(pid).expect("read child start_time");
let recorded = DateTime::from_timestamp(st as i64, 0).unwrap();
let exit = WorkerExit {
code: Some(0),
signal: None,
at: Utc::now(),
};
let n = node(Some(pid as i32), Some(recorded), Some(exit));
assert_eq!(
classify_worker(&n),
WorkerState::Live {
pid,
start_time: st
},
"OS proof of a live original worker must override a stale told exit"
);
let _ = child.kill();
let _ = child.wait();
}
#[test]
fn no_pid_is_no_pid() {
assert_eq!(classify_worker(&node(None, None, None)), WorkerState::NoPid);
assert_eq!(
classify_worker(&node(Some(0), None, None)),
WorkerState::NoPid
);
}
#[test]
fn dead_pid_is_gone() {
let mut child = spawn_sleeper();
let pid = child.id();
child.kill().unwrap();
child.wait().unwrap();
assert_eq!(
classify_worker(&node(Some(pid as i32), None, None)),
WorkerState::Gone
);
}
#[test]
fn live_pid_identity_gates_fenceability() {
let mut child = spawn_sleeper();
let pid = child.id();
let st = watchdog::pid_start_time(pid).expect("read child start_time");
let recorded = DateTime::from_timestamp(st as i64, 0).unwrap();
assert_eq!(
classify_worker(&node(Some(pid as i32), Some(recorded), None)),
WorkerState::Live {
pid,
start_time: st
}
);
assert_eq!(
classify_worker(&node(Some(pid as i32), None, None)),
WorkerState::Unverifiable { pid }
);
let bogus = DateTime::from_timestamp(1, 0).unwrap();
assert_eq!(
classify_worker(&node(Some(pid as i32), Some(bogus), None)),
WorkerState::Gone
);
let _ = child.kill();
let _ = child.wait();
}
#[test]
fn fence_worker_terminates_a_live_process() {
let mut child = spawn_sleeper();
let pid = child.id();
assert!(pid_file::pid_alive(pid));
let st = watchdog::pid_start_time(pid).expect("read child start_time");
let reaper = std::thread::spawn(move || {
let _ = child.wait();
});
fence_worker(pid, st).expect("fence succeeds");
reaper.join().unwrap();
assert!(!pid_file::pid_alive(pid), "worker must be dead after fence");
}
#[test]
fn fence_worker_does_not_signal_a_recycled_pid() {
let mut child = spawn_sleeper();
let pid = child.id();
fence_worker(pid, 1).expect("mismatched identity is a benign no-op");
assert!(
pid_file::pid_alive(pid),
"an identity-mismatched pid must be left untouched"
);
let _ = child.kill();
let _ = child.wait();
}
}