use crate::execution_attempt::{read_attempt_activity, read_attempt_log_tail};
use crate::execution_control::collect_process_ids;
use crate::execution_store::{ExecutionRecord, ExecutionStatus, ExecutionStore};
use crate::exit_reason::{
describe_exit_code, resolve_exit_reason, resolve_memory_exhaustion, CGROUP_OOM_EXIT_REASON,
};
use crate::isolation::isolation_log::FATAL_MARKER_TAIL_BYTES;
use crate::output_blocks::{escape_for_links_notation, format_value_for_links_notation};
use crate::status_footer::{parse_footer_from_tail, read_footer_from_log};
use crate::status_probe::{
apply_end_time, backend_exit_code, is_detached_docker_record, is_detached_session_alive,
read_docker_state, resolve_oom_observation,
};
use chrono::Utc;
use serde_json::Value;
pub fn read_exit_code_from_log(log_path: &str) -> Option<i32> {
read_footer_from_log(log_path).exit_code
}
pub fn enrich_detached_status(record: &ExecutionRecord) -> ExecutionRecord {
let mut enriched =
crate::container_removal::attach_container_removal(resolve_detached_status(record));
if let Some(state) = crate::cpu_penalty_monitor::read_state(record) {
enriched.cpu_penalty = Some(state);
}
read_attempt_activity(&mut enriched);
enriched.attempt_history = enrich_attempt_history(record);
let tail = crate::exit_evidence::record_tail(
&enriched,
read_attempt_log_tail(&enriched, FATAL_MARKER_TAIL_BYTES),
);
attach_exit_diagnostics(enriched, tail.as_deref())
}
fn attach_exit_diagnostics(mut enriched: ExecutionRecord, tail: Option<&str>) -> ExecutionRecord {
if enriched.status != ExecutionStatus::Executed {
return enriched;
}
let oom_kills = enriched.cgroup_memory.and_then(|memory| memory.oom_kills);
if let Some(reason) =
resolve_exit_reason(enriched.exit_code, tail, enriched.oom_killed, oom_kills)
{
enriched.exit_reason = Some(reason);
} else if enriched.exit_reason.as_deref() == Some(CGROUP_OOM_EXIT_REASON) {
enriched.exit_reason = None;
}
if enriched
.memory_exhausted_reason
.as_deref()
.is_some_and(|s| {
s.starts_with("Docker reported")
|| s.starts_with("cgroup memory.events")
|| s.starts_with("Exit-time evidence")
})
{
enriched.memory_exhausted = None;
enriched.memory_exhausted_reason = None;
}
if enriched.memory_exhausted.is_none() {
if let Some(memory) =
resolve_memory_exhaustion(enriched.exit_code, tail, enriched.oom_killed, oom_kills)
{
enriched.memory_exhausted = Some(memory.memory_exhausted);
enriched.memory_exhausted_reason = Some(memory.memory_exhausted_reason);
}
}
enriched
}
fn enrich_attempt_history(record: &ExecutionRecord) -> Vec<Value> {
let mut base = record.clone();
base.attempt_history.clear();
record
.attempt_history
.iter()
.enumerate()
.map(|(index, attempt)| {
let mut historical = base.clone();
historical.status = if attempt.get("status").and_then(Value::as_str) == Some("executed")
{
ExecutionStatus::Executed
} else {
ExecutionStatus::Executing
};
historical.exit_code = attempt
.get("exitCode")
.and_then(Value::as_i64)
.and_then(|n| i32::try_from(n).ok());
historical.oom_killed = attempt.get("oomKilled").and_then(Value::as_bool);
historical.cgroup_memory = attempt
.get("cgroupMemory")
.and_then(|v| serde_json::from_value(v.clone()).ok());
historical.exit_reason = attempt
.get("exitReason")
.and_then(Value::as_str)
.map(str::to_string);
historical.memory_exhausted = attempt.get("memoryExhausted").and_then(Value::as_bool);
historical.memory_exhausted_reason = attempt
.get("memoryExhaustedReason")
.and_then(Value::as_str)
.map(str::to_string);
historical.options.remove("exitEvidence");
if let Some(evidence) = attempt.get("exitEvidence") {
historical
.options
.insert("exitEvidence".into(), evidence.clone());
}
let end = if let Some(next) = record.attempt_history.get(index + 1) {
next.get("logOffset").and_then(Value::as_u64)
} else {
record.attempt.as_ref().and_then(|a| a.log_offset)
};
let tail = attempt
.get("logOffset")
.and_then(Value::as_u64)
.zip(end)
.filter(|(start, end)| end >= start)
.and_then(|(start, end)| {
crate::isolation::isolation_log::read_log_tail_range(
&record.log_path,
FATAL_MARKER_TAIL_BYTES,
start,
end,
)
});
let tail = crate::exit_evidence::record_tail(&historical, tail);
let enriched = attach_exit_diagnostics(historical, tail.as_deref()).to_json();
let mut output = attempt.clone();
for key in ["exitReason", "memoryExhausted", "memoryExhaustedReason"] {
if let Some(object) = output.as_object_mut() {
object.remove(key);
}
if let Some(value) = enriched.get(key) {
output[key] = value.clone();
}
}
output
})
.collect()
}
fn resolve_detached_status(record: &ExecutionRecord) -> ExecutionRecord {
if crate::launch_owner::has_active_launch(record) {
return record.clone();
}
let tail = read_attempt_log_tail(record, FATAL_MARKER_TAIL_BYTES);
let footer = parse_footer_from_tail(tail.as_deref().unwrap_or(""));
let footer_exit = footer.exit_code;
let is_detached_docker = is_detached_docker_record(record);
let docker_state = if is_detached_docker {
read_docker_state(record)
} else {
None
};
let oom_killed = resolve_oom_observation(record, docker_state.as_ref());
let clone_record = || {
let mut enriched = record.clone();
if oom_killed.is_some() {
enriched.oom_killed = oom_killed;
}
enriched
};
let alive = if is_detached_docker {
docker_state.as_ref().map(|state| state.running)
} else {
is_detached_session_alive(record)
};
let alive = match alive {
Some(value) => value,
None => {
let is_detached =
record.options.get("isolationMode").and_then(|v| v.as_str()) == Some("detached");
if is_detached && record.status == ExecutionStatus::Executing {
if footer_exit.is_some() {
let mut enriched = clone_record();
enriched.status = ExecutionStatus::Executed;
enriched.exit_code = footer_exit;
apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
return enriched;
}
if oom_killed == Some(true) {
let mut enriched = clone_record();
enriched.status = ExecutionStatus::Executed;
enriched.exit_code = Some(137);
apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
return enriched;
}
}
return clone_record();
}
};
let mut enriched = clone_record();
if alive && enriched.status == ExecutionStatus::Executed {
if enriched.exit_code.is_none() && footer_exit.is_none() {
enriched.status = ExecutionStatus::Executing;
enriched.exit_code = None;
enriched.end_time = None;
enriched.end_time_source = None;
}
} else if !alive && enriched.status == ExecutionStatus::Executing {
enriched.status = ExecutionStatus::Executed;
if enriched.exit_code.is_none() {
enriched.exit_code = Some(
backend_exit_code(docker_state.as_ref())
.or(footer_exit)
.or(if oom_killed == Some(true) {
Some(137)
} else {
None
})
.unwrap_or(-1),
);
}
apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
}
enriched
}
pub fn attach_current_time(record: &ExecutionRecord) -> Option<String> {
if record.status == ExecutionStatus::Executing {
Some(Utc::now().to_rfc3339())
} else {
None
}
}
pub fn format_record_as_links_notation(record: &ExecutionRecord) -> String {
format_record_as_links_notation_with_current_time(record, None)
}
pub fn format_record_as_links_notation_with_current_time(
record: &ExecutionRecord,
current_time: Option<&str>,
) -> String {
format_record_as_links_notation_with_enrichments(record, current_time, None, &[])
}
fn append_links_array(lines: &mut Vec<String>, values: &[Value], indent: usize) {
let prefix = " ".repeat(indent);
if values.is_empty() {
lines.push(format!("{}()", prefix));
return;
}
lines.push(format!("{}(", prefix));
for value in values {
match value {
Value::Array(nested) => append_links_array(lines, nested, indent + 2),
Value::Object(map) => {
for (child_key, child_value) in map {
if !child_value.is_null() {
append_links_value(lines, child_key, child_value, indent + 2);
}
}
}
_ => lines.push(format!(
"{}{}",
" ".repeat(indent + 2),
format_value_for_links_notation(value)
)),
}
}
lines.push(format!("{})", prefix));
}
fn append_links_value(lines: &mut Vec<String>, key: &str, value: &Value, indent: usize) {
let prefix = " ".repeat(indent);
match value {
Value::Object(map) => {
if map.is_empty() {
return;
}
lines.push(format!("{}{}", prefix, key));
for (child_key, child_value) in map {
if !child_value.is_null() {
append_links_value(lines, child_key, child_value, indent + 4);
}
}
}
Value::Array(values) => {
lines.push(format!("{}{}", prefix, key));
append_links_array(lines, values, indent + 2);
}
_ => lines.push(format!(
"{}{} {}",
prefix,
key,
format_value_for_links_notation(value)
)),
}
}
fn format_record_as_links_notation_with_enrichments(
record: &ExecutionRecord,
current_time: Option<&str>,
process_ids: Option<&Value>,
identity: &[(String, Value)],
) -> String {
let json = record.to_json();
let mut lines = vec![record.uuid.clone()];
if let Value::Object(map) = json {
for (key, value) in map {
if !value.is_null() {
if key == "options" {
if let Value::Object(opts) = &value {
if !opts.is_empty() {
lines.push(" options".to_string());
for (opt_key, opt_value) in opts {
if !opt_value.is_null() {
append_links_value(&mut lines, opt_key, opt_value, 4);
}
}
}
}
} else if value.is_object() || value.is_array() {
append_links_value(&mut lines, &key, &value, 2);
} else {
let formatted_value = match &value {
Value::String(s) => escape_for_links_notation(s),
Value::Bool(b) => b.to_string(),
Value::Number(n) => n.to_string(),
Value::Null => "null".to_string(),
Value::Object(_) | Value::Array(_) => {
format_value_for_links_notation(&value)
}
};
lines.push(format!(" {} {}", key, formatted_value));
}
}
if key == "uuid" {
for (field, value) in identity {
append_links_value(&mut lines, field, value, 2);
}
}
if key == "pid" {
if let Some(process_ids) = process_ids {
append_links_value(&mut lines, "processIds", process_ids, 2);
}
}
if key == "startTime" {
if let Some(ct) = current_time {
lines.push(format!(" currentTime {}", escape_for_links_notation(ct)));
}
}
}
}
lines.join("\n")
}
pub fn format_record_as_text(record: &ExecutionRecord) -> String {
format_record_as_text_with_current_time(record, None)
}
pub fn format_record_as_text_with_current_time(
record: &ExecutionRecord,
current_time: Option<&str>,
) -> String {
format_record_as_text_with_enrichments(record, current_time, None, &[])
}
fn append_text_process_ids(lines: &mut Vec<String>, process_ids: &Value) {
let Value::Object(map) = process_ids else {
return;
};
if map.is_empty() {
return;
}
lines.push("Process IDs:".to_string());
for (key, value) in map {
let value_str = match value {
Value::String(s) => s.clone(),
Value::Bool(b) => b.to_string(),
Value::Number(n) => n.to_string(),
Value::Null => "null".to_string(),
other => serde_json::to_string(other).unwrap_or_default(),
};
lines.push(format!(" {}: {}", key, value_str));
}
}
fn format_record_as_text_with_enrichments(
record: &ExecutionRecord,
current_time: Option<&str>,
process_ids: Option<&Value>,
identity: &[(String, Value)],
) -> String {
let exit_code_str = record
.exit_code
.map(|c| describe_exit_code(Some(c)).text)
.unwrap_or_else(|| "N/A".to_string());
let pid_str = record
.pid
.map(|p| p.to_string())
.unwrap_or_else(|| "N/A".to_string());
let end_time_str = record.end_time.as_deref().unwrap_or("N/A");
let mut lines = vec![
"Execution Status".to_string(),
"=".repeat(50),
format!("UUID: {}", record.uuid),
format!("Status: {}", record.status),
format!("Command: {}", record.command),
];
lines.extend(crate::execution_chain::identity_text_lines(
record, identity,
));
lines.push(format!("Exit Code: {}", exit_code_str));
if let Some(oom_killed) = record.oom_killed {
lines.push(format!("OOM Killed: {}", oom_killed));
}
if let Some(ref exit_reason) = record.exit_reason {
lines.push(format!("Exit Reason: {}", exit_reason));
}
if let Some(memory_exhausted) = record.memory_exhausted {
lines.push(format!("Memory Exhausted: {}", memory_exhausted));
}
if let Some(ref reason) = record.memory_exhausted_reason {
lines.push(format!("Memory Evidence: {}", reason));
}
if let Some(ref memory) = record.cgroup_memory {
lines.push(format!(
"Cgroup Memory: {}",
crate::cgroup_memory::format_cgroup_memory(memory)
));
}
if let Some(ref state) = record.cpu_penalty {
lines.push(format!("CPU Penalty: {}", state));
}
lines.push(format!("PID: {}", pid_str));
if let Some(process_ids) = process_ids {
append_text_process_ids(&mut lines, process_ids);
}
lines.extend([
format!("Working Directory: {}", record.working_directory),
format!("Shell: {}", record.shell),
format!("Platform: {}", record.platform),
format!("Start Time: {}", record.start_time),
]);
if let Some(ct) = current_time {
lines.push(format!("Current Time: {}", ct));
}
lines.push(format!("End Time: {}", end_time_str));
if let Some(ref source) = record.end_time_source {
lines.push(format!("End Time Source: {}", source));
}
if let Some(ref started) = record.container_started_at {
lines.push(format!("Container Started: {}", started));
}
if let Some(ref detected) = record.stale_detected_at {
lines.push(format!("Stale Detected At: {}", detected));
}
if let Some(ref state) = record.container_state {
lines.push(format!("Container State: {}", state));
lines.push(format!(
"Container Removed: {}",
record
.container_removed_observed_at
.as_deref()
.unwrap_or("N/A")
));
}
lines.push(format!("Log Path: {}", record.log_path));
if let Some(attempt) = &record.attempt {
lines.extend([
format!("Attempt: {}", attempt.number),
format!("Attempt Started: {}", attempt.started_at),
format!(
"Attempt Log Offset: {}",
attempt
.log_offset
.map(|offset| offset.to_string())
.unwrap_or_else(|| "N/A".into())
),
format!(
"Launch Accepted: {}",
attempt.launch_accepted_at.as_deref().unwrap_or("N/A")
),
format!(
"Watcher Attached: {}",
attempt.watcher_attached_at.as_deref().unwrap_or("N/A")
),
format!(
"Last Output: {}",
attempt.last_output_at.as_deref().unwrap_or("N/A")
),
]);
}
if !record.options.is_empty() {
lines.push("Options:".to_string());
let mut options: Vec<_> = record.options.iter().collect();
options.sort_by(|a, b| a.0.cmp(b.0));
for (key, value) in options {
let value_str = match value {
Value::String(s) => s.clone(),
Value::Bool(b) => b.to_string(),
Value::Number(n) => n.to_string(),
Value::Null => "null".to_string(),
other => serde_json::to_string(other).unwrap_or_default(),
};
lines.push(format!(" {}: {}", key, value_str));
}
}
lines.join("\n")
}
fn record_json_with_enrichments(
record: &ExecutionRecord,
current_time: Option<&str>,
process_ids: Option<&Value>,
identity: &[(String, Value)],
) -> Value {
let mut json = record.to_json();
if let Value::Object(map) = &mut json {
map.extend(identity.iter().cloned());
if let Some(process_ids) = process_ids {
map.insert("processIds".to_string(), process_ids.clone());
}
if let Some(ct) = current_time {
map.insert("currentTime".to_string(), Value::String(ct.to_string()));
}
}
json
}
pub fn format_record(record: &ExecutionRecord, format: &str) -> Result<String, String> {
format_record_with_current_time(record, format, None)
}
pub fn format_record_with_current_time(
record: &ExecutionRecord,
format: &str,
current_time: Option<&str>,
) -> Result<String, String> {
format_record_with_enrichments(record, format, current_time, None, &[])
}
fn format_record_with_enrichments(
record: &ExecutionRecord,
format: &str,
current_time: Option<&str>,
process_ids: Option<&Value>,
identity: &[(String, Value)],
) -> Result<String, String> {
match format {
"links-notation" => Ok(format_record_as_links_notation_with_enrichments(
record,
current_time,
process_ids,
identity,
)),
"json" => serde_json::to_string_pretty(&record_json_with_enrichments(
record,
current_time,
process_ids,
identity,
))
.map_err(|e| format!("Failed to serialize to JSON: {}", e)),
"text" => Ok(format_record_as_text_with_enrichments(
record,
current_time,
process_ids,
identity,
)),
_ => Err(format!("Unknown output format: {}", format)),
}
}
fn sort_records_by_start_time_desc(records: &mut [ExecutionRecord]) {
records.sort_by(|a, b| b.start_time.cmp(&a.start_time));
}
fn indent_block(block: &str, spaces: usize) -> String {
let prefix = " ".repeat(spaces);
block
.lines()
.map(|line| format!("{}{}", prefix, line))
.collect::<Vec<_>>()
.join("\n")
}
pub fn format_record_list_as_links_notation(records: &[ExecutionRecord]) -> String {
let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
let process_ids = vec![None; records.len()];
format_record_list_as_links_notation_with_current_times(records, ¤t_times, &process_ids)
}
fn format_record_list_as_links_notation_with_current_times(
records: &[ExecutionRecord],
current_times: &[Option<String>],
process_ids: &[Option<Value>],
) -> String {
let mut lines = vec![
"executions".to_string(),
format!(" count {}", records.len()),
];
if records.is_empty() {
lines.push(" records ()".to_string());
return lines.join("\n");
}
lines.push(" records".to_string());
for ((record, current_time), process_ids) in records
.iter()
.zip(current_times.iter())
.zip(process_ids.iter())
{
let block = format_record_as_links_notation_with_enrichments(
record,
current_time.as_deref(),
process_ids.as_ref(),
&[],
);
lines.push(indent_block(&block, 4));
}
lines.join("\n")
}
pub fn format_record_list_as_text(records: &[ExecutionRecord]) -> String {
let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
let process_ids = vec![None; records.len()];
format_record_list_as_text_with_current_times(records, ¤t_times, &process_ids)
}
fn format_record_list_as_text_with_current_times(
records: &[ExecutionRecord],
current_times: &[Option<String>],
process_ids: &[Option<Value>],
) -> String {
let mut lines = vec![
"Executions".to_string(),
"=".repeat(50),
format!("Count: {}", records.len()),
];
for ((record, current_time), process_ids) in records
.iter()
.zip(current_times.iter())
.zip(process_ids.iter())
{
lines.push(String::new());
lines.push(format_record_as_text_with_enrichments(
record,
current_time.as_deref(),
process_ids.as_ref(),
&[],
));
}
lines.join("\n")
}
fn record_list_json_with_current_times(
records: &[ExecutionRecord],
current_times: &[Option<String>],
process_ids: &[Option<Value>],
) -> Value {
let executions: Vec<Value> = records
.iter()
.zip(current_times.iter())
.zip(process_ids.iter())
.map(|((record, current_time), process_ids)| {
record_json_with_enrichments(record, current_time.as_deref(), process_ids.as_ref(), &[])
})
.collect();
serde_json::json!({
"count": records.len(),
"executions": executions,
})
}
pub fn format_record_list(records: &[ExecutionRecord], format: &str) -> Result<String, String> {
let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
let process_ids = vec![None; records.len()];
format_record_list_with_current_times(records, format, ¤t_times, &process_ids)
}
fn format_record_list_with_current_times(
records: &[ExecutionRecord],
format: &str,
current_times: &[Option<String>],
process_ids: &[Option<Value>],
) -> Result<String, String> {
match format {
"links-notation" => Ok(format_record_list_as_links_notation_with_current_times(
records,
current_times,
process_ids,
)),
"json" => serde_json::to_string_pretty(&record_list_json_with_current_times(
records,
current_times,
process_ids,
))
.map_err(|e| format!("Failed to serialize to JSON: {}", e)),
"text" => Ok(format_record_list_as_text_with_current_times(
records,
current_times,
process_ids,
)),
_ => Err(format!("Unknown output format: {}", format)),
}
}
pub struct StatusQueryResult {
pub success: bool,
pub output: Option<String>,
pub error: Option<String>,
}
pub fn list_executions(
store: Option<&ExecutionStore>,
output_format: Option<&str>,
) -> StatusQueryResult {
list_executions_filtered(store, output_format, false)
}
pub fn list_executions_filtered(
store: Option<&ExecutionStore>,
output_format: Option<&str>,
running_only: bool,
) -> StatusQueryResult {
let store = match store {
Some(s) => s,
None => {
return StatusQueryResult {
success: false,
output: None,
error: Some("Execution tracking is disabled.".to_string()),
}
}
};
let mut records: Vec<ExecutionRecord> =
store.get_all().iter().map(enrich_detached_status).collect();
if running_only {
records.retain(|record| record.status == ExecutionStatus::Executing);
}
sort_records_by_start_time_desc(&mut records);
let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
let process_ids: Vec<Option<Value>> = records.iter().map(collect_process_ids).collect();
let format = output_format.unwrap_or("links-notation");
match format_record_list_with_current_times(&records, format, ¤t_times, &process_ids) {
Ok(output) => StatusQueryResult {
success: true,
output: Some(output),
error: None,
},
Err(e) => StatusQueryResult {
success: false,
output: None,
error: Some(e),
},
}
}
pub fn query_status(
store: Option<&ExecutionStore>,
identifier: &str,
output_format: Option<&str>,
) -> StatusQueryResult {
let store = match store {
Some(s) => s,
None => {
return StatusQueryResult {
success: false,
output: None,
error: Some("Execution tracking is disabled.".to_string()),
}
}
};
let record = match store.get(identifier) {
Some(r) => r,
None => {
return StatusQueryResult {
success: false,
output: None,
error: Some(format!(
"No execution found with UUID or session name: {}",
identifier
)),
}
}
};
let enriched = enrich_detached_status(&record);
crate::container_removal::persist_container_removal(store, &record, &enriched);
let current_time = attach_current_time(&enriched);
let process_ids = collect_process_ids(&enriched);
let identity = crate::execution_chain::status_identity(&enriched, Some(store));
let format = output_format.unwrap_or("links-notation");
match format_record_with_enrichments(
&enriched,
format,
current_time.as_deref(),
process_ids.as_ref(),
&identity,
) {
Ok(output) => StatusQueryResult {
success: true,
output: Some(output),
error: None,
},
Err(e) => StatusQueryResult {
success: false,
output: None,
error: Some(e),
},
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::execution_store::ExecutionRecordOptions;
use serde_json::json;
fn executing_record() -> ExecutionRecord {
ExecutionRecord::with_options(ExecutionRecordOptions {
command: "sleep 60".to_string(),
uuid: Some("issue-126-rust".to_string()),
pid: Some(667105),
status: Some(ExecutionStatus::Executing),
log_path: Some("/tmp/issue-126.log".to_string()),
start_time: Some("2026-04-23T10:00:00Z".to_string()),
working_directory: Some("/home/user".to_string()),
shell: Some("/bin/bash".to_string()),
platform: Some("linux".to_string()),
..Default::default()
})
}
#[test]
fn links_notation_indents_nested_process_id_arrays() {
let process_ids = json!({
"wrapperPid": 667105,
"screenPid": 667120,
"commandPids": [667121, 667122],
});
let output = format_record_with_enrichments(
&executing_record(),
"links-notation",
Some("2026-04-23T10:10:13.042Z"),
Some(&process_ids),
&[],
)
.expect("links-notation should format");
assert!(
output.contains(
" commandPids\n (\n 667121\n 667122\n )"
),
"processIds should be a nested indented block, output: {}",
output
);
assert!(
!output.contains("\n(\n"),
"opening parenthesis must not start at column 1: {}",
output
);
}
}