use crate::execution_attempt::{read_attempt_activity, read_attempt_log_tail};
use crate::execution_control::collect_process_ids;
use crate::execution_store::{ExecutionRecord, ExecutionStatus, ExecutionStore};
use crate::exit_reason::{
describe_exit_code, resolve_exit_reason, resolve_memory_exhaustion, CGROUP_OOM_EXIT_REASON,
};
use crate::isolation::isolation_log::FATAL_MARKER_TAIL_BYTES;
use crate::output_blocks::{escape_for_links_notation, format_value_for_links_notation};
use crate::status_footer::{parse_footer_from_tail, read_footer_from_log};
use crate::status_probe::{
apply_end_time, backend_exit_code, is_detached_docker_record, is_detached_session_alive,
read_docker_state, resolve_oom_observation,
};
use chrono::Utc;
use serde_json::Value;
pub fn read_exit_code_from_log(log_path: &str) -> Option<i32> {
read_footer_from_log(log_path).exit_code
}
pub fn enrich_detached_status(record: &ExecutionRecord) -> ExecutionRecord {
let mut enriched = resolve_detached_status(record);
read_attempt_activity(&mut enriched);
if enriched.status != ExecutionStatus::Executed {
return enriched;
}
let tail = read_attempt_log_tail(&enriched, FATAL_MARKER_TAIL_BYTES);
let oom_kills = enriched.cgroup_memory.and_then(|memory| memory.oom_kills);
if enriched.exit_reason.is_none()
|| enriched.exit_reason.as_deref() == Some(CGROUP_OOM_EXIT_REASON)
{
enriched.exit_reason = resolve_exit_reason(
enriched.exit_code,
tail.as_deref(),
enriched.oom_killed,
oom_kills,
);
}
if enriched.memory_exhausted.is_none() {
if let Some(memory) = resolve_memory_exhaustion(
enriched.exit_code,
tail.as_deref(),
enriched.oom_killed,
oom_kills,
) {
enriched.memory_exhausted = Some(memory.memory_exhausted);
enriched.memory_exhausted_reason = Some(memory.memory_exhausted_reason);
}
}
enriched
}
fn resolve_detached_status(record: &ExecutionRecord) -> ExecutionRecord {
let tail = read_attempt_log_tail(record, FATAL_MARKER_TAIL_BYTES);
let footer = parse_footer_from_tail(tail.as_deref().unwrap_or(""));
let footer_exit = footer.exit_code;
let is_detached_docker = is_detached_docker_record(record);
let docker_state = if is_detached_docker {
read_docker_state(record)
} else {
None
};
let oom_killed = resolve_oom_observation(record, docker_state.as_ref());
let clone_record = || {
let mut enriched = record.clone();
if oom_killed.is_some() {
enriched.oom_killed = oom_killed;
}
enriched
};
let alive = if is_detached_docker {
docker_state.as_ref().map(|state| state.running)
} else {
is_detached_session_alive(record)
};
let alive = match alive {
Some(value) => value,
None => {
let is_detached =
record.options.get("isolationMode").and_then(|v| v.as_str()) == Some("detached");
if is_detached && record.status == ExecutionStatus::Executing {
if footer_exit.is_some() {
let mut enriched = clone_record();
enriched.status = ExecutionStatus::Executed;
enriched.exit_code = footer_exit;
apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
return enriched;
}
if oom_killed == Some(true) {
let mut enriched = clone_record();
enriched.status = ExecutionStatus::Executed;
enriched.exit_code = Some(137);
apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
return enriched;
}
}
return clone_record();
}
};
let mut enriched = clone_record();
if alive && enriched.status == ExecutionStatus::Executed {
if enriched.exit_code.is_none() && footer_exit.is_none() {
enriched.status = ExecutionStatus::Executing;
enriched.exit_code = None;
enriched.end_time = None;
enriched.end_time_source = None;
}
} else if !alive && enriched.status == ExecutionStatus::Executing {
enriched.status = ExecutionStatus::Executed;
if enriched.exit_code.is_none() {
enriched.exit_code = Some(
backend_exit_code(docker_state.as_ref())
.or(footer_exit)
.or(if oom_killed == Some(true) {
Some(137)
} else {
None
})
.unwrap_or(-1),
);
}
apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
}
enriched
}
pub fn attach_current_time(record: &ExecutionRecord) -> Option<String> {
if record.status == ExecutionStatus::Executing {
Some(Utc::now().to_rfc3339())
} else {
None
}
}
pub fn format_record_as_links_notation(record: &ExecutionRecord) -> String {
format_record_as_links_notation_with_current_time(record, None)
}
pub fn format_record_as_links_notation_with_current_time(
record: &ExecutionRecord,
current_time: Option<&str>,
) -> String {
format_record_as_links_notation_with_enrichments(record, current_time, None)
}
fn append_links_array(lines: &mut Vec<String>, values: &[Value], indent: usize) {
let prefix = " ".repeat(indent);
if values.is_empty() {
lines.push(format!("{}()", prefix));
return;
}
lines.push(format!("{}(", prefix));
for value in values {
match value {
Value::Array(nested) => append_links_array(lines, nested, indent + 2),
Value::Object(map) => {
for (child_key, child_value) in map {
if !child_value.is_null() {
append_links_value(lines, child_key, child_value, indent + 2);
}
}
}
_ => lines.push(format!(
"{}{}",
" ".repeat(indent + 2),
format_value_for_links_notation(value)
)),
}
}
lines.push(format!("{})", prefix));
}
fn append_links_value(lines: &mut Vec<String>, key: &str, value: &Value, indent: usize) {
let prefix = " ".repeat(indent);
match value {
Value::Object(map) => {
if map.is_empty() {
return;
}
lines.push(format!("{}{}", prefix, key));
for (child_key, child_value) in map {
if !child_value.is_null() {
append_links_value(lines, child_key, child_value, indent + 4);
}
}
}
Value::Array(values) => {
lines.push(format!("{}{}", prefix, key));
append_links_array(lines, values, indent + 2);
}
_ => lines.push(format!(
"{}{} {}",
prefix,
key,
format_value_for_links_notation(value)
)),
}
}
fn format_record_as_links_notation_with_enrichments(
record: &ExecutionRecord,
current_time: Option<&str>,
process_ids: Option<&Value>,
) -> String {
let json = record.to_json();
let mut lines = vec![record.uuid.clone()];
if let Value::Object(map) = json {
for (key, value) in map {
if !value.is_null() {
if key == "options" {
if let Value::Object(opts) = &value {
if !opts.is_empty() {
lines.push(" options".to_string());
for (opt_key, opt_value) in opts {
if !opt_value.is_null() {
let formatted = format_value_for_links_notation(opt_value);
lines.push(format!(" {} {}", opt_key, formatted));
}
}
}
}
} else if value.is_object() {
append_links_value(&mut lines, &key, &value, 2);
} else {
let formatted_value = match &value {
Value::String(s) => escape_for_links_notation(s),
Value::Bool(b) => b.to_string(),
Value::Number(n) => n.to_string(),
Value::Null => "null".to_string(),
Value::Object(_) | Value::Array(_) => {
format_value_for_links_notation(&value)
}
};
lines.push(format!(" {} {}", key, formatted_value));
}
}
if key == "pid" {
if let Some(process_ids) = process_ids {
append_links_value(&mut lines, "processIds", process_ids, 2);
}
}
if key == "startTime" {
if let Some(ct) = current_time {
lines.push(format!(" currentTime {}", escape_for_links_notation(ct)));
}
}
}
}
lines.join("\n")
}
pub fn format_record_as_text(record: &ExecutionRecord) -> String {
format_record_as_text_with_current_time(record, None)
}
pub fn format_record_as_text_with_current_time(
record: &ExecutionRecord,
current_time: Option<&str>,
) -> String {
format_record_as_text_with_enrichments(record, current_time, None)
}
fn append_text_process_ids(lines: &mut Vec<String>, process_ids: &Value) {
let Value::Object(map) = process_ids else {
return;
};
if map.is_empty() {
return;
}
lines.push("Process IDs:".to_string());
for (key, value) in map {
let value_str = match value {
Value::String(s) => s.clone(),
Value::Bool(b) => b.to_string(),
Value::Number(n) => n.to_string(),
Value::Null => "null".to_string(),
other => serde_json::to_string(other).unwrap_or_default(),
};
lines.push(format!(" {}: {}", key, value_str));
}
}
fn format_record_as_text_with_enrichments(
record: &ExecutionRecord,
current_time: Option<&str>,
process_ids: Option<&Value>,
) -> String {
let exit_code_str = record
.exit_code
.map(|c| describe_exit_code(Some(c)).text)
.unwrap_or_else(|| "N/A".to_string());
let pid_str = record
.pid
.map(|p| p.to_string())
.unwrap_or_else(|| "N/A".to_string());
let end_time_str = record.end_time.as_deref().unwrap_or("N/A");
let mut lines = vec![
"Execution Status".to_string(),
"=".repeat(50),
format!("UUID: {}", record.uuid),
format!("Status: {}", record.status),
format!("Command: {}", record.command),
format!("Exit Code: {}", exit_code_str),
];
if let Some(oom_killed) = record.oom_killed {
lines.push(format!("OOM Killed: {}", oom_killed));
}
if let Some(ref exit_reason) = record.exit_reason {
lines.push(format!("Exit Reason: {}", exit_reason));
}
if let Some(memory_exhausted) = record.memory_exhausted {
lines.push(format!("Memory Exhausted: {}", memory_exhausted));
}
if let Some(ref reason) = record.memory_exhausted_reason {
lines.push(format!("Memory Evidence: {}", reason));
}
if let Some(ref memory) = record.cgroup_memory {
lines.push(format!(
"Cgroup Memory: {}",
crate::cgroup_memory::format_cgroup_memory(memory)
));
}
lines.push(format!("PID: {}", pid_str));
if let Some(process_ids) = process_ids {
append_text_process_ids(&mut lines, process_ids);
}
lines.extend([
format!("Working Directory: {}", record.working_directory),
format!("Shell: {}", record.shell),
format!("Platform: {}", record.platform),
format!("Start Time: {}", record.start_time),
]);
if let Some(ct) = current_time {
lines.push(format!("Current Time: {}", ct));
}
lines.push(format!("End Time: {}", end_time_str));
if let Some(ref source) = record.end_time_source {
lines.push(format!("End Time Source: {}", source));
}
if let Some(ref started) = record.container_started_at {
lines.push(format!("Container Started: {}", started));
}
if let Some(ref detected) = record.stale_detected_at {
lines.push(format!("Stale Detected At: {}", detected));
}
lines.push(format!("Log Path: {}", record.log_path));
if let Some(attempt) = &record.attempt {
lines.extend([
format!("Attempt: {}", attempt.number),
format!("Attempt Started: {}", attempt.started_at),
format!(
"Attempt Log Offset: {}",
attempt
.log_offset
.map(|offset| offset.to_string())
.unwrap_or_else(|| "N/A".into())
),
format!(
"Launch Accepted: {}",
attempt.launch_accepted_at.as_deref().unwrap_or("N/A")
),
format!(
"Watcher Attached: {}",
attempt.watcher_attached_at.as_deref().unwrap_or("N/A")
),
format!(
"Last Output: {}",
attempt.last_output_at.as_deref().unwrap_or("N/A")
),
]);
}
if !record.options.is_empty() {
lines.push("Options:".to_string());
for (key, value) in &record.options {
let value_str = match value {
Value::String(s) => s.clone(),
Value::Bool(b) => b.to_string(),
Value::Number(n) => n.to_string(),
Value::Null => "null".to_string(),
other => serde_json::to_string(other).unwrap_or_default(),
};
lines.push(format!(" {}: {}", key, value_str));
}
}
lines.join("\n")
}
fn record_json_with_enrichments(
record: &ExecutionRecord,
current_time: Option<&str>,
process_ids: Option<&Value>,
) -> Value {
let mut json = record.to_json();
if let Value::Object(map) = &mut json {
if let Some(process_ids) = process_ids {
map.insert("processIds".to_string(), process_ids.clone());
}
if let Some(ct) = current_time {
map.insert("currentTime".to_string(), Value::String(ct.to_string()));
}
}
json
}
pub fn format_record(record: &ExecutionRecord, format: &str) -> Result<String, String> {
format_record_with_current_time(record, format, None)
}
pub fn format_record_with_current_time(
record: &ExecutionRecord,
format: &str,
current_time: Option<&str>,
) -> Result<String, String> {
format_record_with_enrichments(record, format, current_time, None)
}
fn format_record_with_enrichments(
record: &ExecutionRecord,
format: &str,
current_time: Option<&str>,
process_ids: Option<&Value>,
) -> Result<String, String> {
match format {
"links-notation" => Ok(format_record_as_links_notation_with_enrichments(
record,
current_time,
process_ids,
)),
"json" => serde_json::to_string_pretty(&record_json_with_enrichments(
record,
current_time,
process_ids,
))
.map_err(|e| format!("Failed to serialize to JSON: {}", e)),
"text" => Ok(format_record_as_text_with_enrichments(
record,
current_time,
process_ids,
)),
_ => Err(format!("Unknown output format: {}", format)),
}
}
fn sort_records_by_start_time_desc(records: &mut [ExecutionRecord]) {
records.sort_by(|a, b| b.start_time.cmp(&a.start_time));
}
fn indent_block(block: &str, spaces: usize) -> String {
let prefix = " ".repeat(spaces);
block
.lines()
.map(|line| format!("{}{}", prefix, line))
.collect::<Vec<_>>()
.join("\n")
}
pub fn format_record_list_as_links_notation(records: &[ExecutionRecord]) -> String {
let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
let process_ids = vec![None; records.len()];
format_record_list_as_links_notation_with_current_times(records, ¤t_times, &process_ids)
}
fn format_record_list_as_links_notation_with_current_times(
records: &[ExecutionRecord],
current_times: &[Option<String>],
process_ids: &[Option<Value>],
) -> String {
let mut lines = vec![
"executions".to_string(),
format!(" count {}", records.len()),
];
if records.is_empty() {
lines.push(" records ()".to_string());
return lines.join("\n");
}
lines.push(" records".to_string());
for ((record, current_time), process_ids) in records
.iter()
.zip(current_times.iter())
.zip(process_ids.iter())
{
let block = format_record_as_links_notation_with_enrichments(
record,
current_time.as_deref(),
process_ids.as_ref(),
);
lines.push(indent_block(&block, 4));
}
lines.join("\n")
}
pub fn format_record_list_as_text(records: &[ExecutionRecord]) -> String {
let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
let process_ids = vec![None; records.len()];
format_record_list_as_text_with_current_times(records, ¤t_times, &process_ids)
}
fn format_record_list_as_text_with_current_times(
records: &[ExecutionRecord],
current_times: &[Option<String>],
process_ids: &[Option<Value>],
) -> String {
let mut lines = vec![
"Executions".to_string(),
"=".repeat(50),
format!("Count: {}", records.len()),
];
for ((record, current_time), process_ids) in records
.iter()
.zip(current_times.iter())
.zip(process_ids.iter())
{
lines.push(String::new());
lines.push(format_record_as_text_with_enrichments(
record,
current_time.as_deref(),
process_ids.as_ref(),
));
}
lines.join("\n")
}
fn record_list_json_with_current_times(
records: &[ExecutionRecord],
current_times: &[Option<String>],
process_ids: &[Option<Value>],
) -> Value {
let executions: Vec<Value> = records
.iter()
.zip(current_times.iter())
.zip(process_ids.iter())
.map(|((record, current_time), process_ids)| {
record_json_with_enrichments(record, current_time.as_deref(), process_ids.as_ref())
})
.collect();
serde_json::json!({
"count": records.len(),
"executions": executions,
})
}
pub fn format_record_list(records: &[ExecutionRecord], format: &str) -> Result<String, String> {
let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
let process_ids = vec![None; records.len()];
format_record_list_with_current_times(records, format, ¤t_times, &process_ids)
}
fn format_record_list_with_current_times(
records: &[ExecutionRecord],
format: &str,
current_times: &[Option<String>],
process_ids: &[Option<Value>],
) -> Result<String, String> {
match format {
"links-notation" => Ok(format_record_list_as_links_notation_with_current_times(
records,
current_times,
process_ids,
)),
"json" => serde_json::to_string_pretty(&record_list_json_with_current_times(
records,
current_times,
process_ids,
))
.map_err(|e| format!("Failed to serialize to JSON: {}", e)),
"text" => Ok(format_record_list_as_text_with_current_times(
records,
current_times,
process_ids,
)),
_ => Err(format!("Unknown output format: {}", format)),
}
}
pub struct StatusQueryResult {
pub success: bool,
pub output: Option<String>,
pub error: Option<String>,
}
pub fn list_executions(
store: Option<&ExecutionStore>,
output_format: Option<&str>,
) -> StatusQueryResult {
list_executions_filtered(store, output_format, false)
}
pub fn list_executions_filtered(
store: Option<&ExecutionStore>,
output_format: Option<&str>,
running_only: bool,
) -> StatusQueryResult {
let store = match store {
Some(s) => s,
None => {
return StatusQueryResult {
success: false,
output: None,
error: Some("Execution tracking is disabled.".to_string()),
}
}
};
let mut records: Vec<ExecutionRecord> =
store.get_all().iter().map(enrich_detached_status).collect();
if running_only {
records.retain(|record| record.status == ExecutionStatus::Executing);
}
sort_records_by_start_time_desc(&mut records);
let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
let process_ids: Vec<Option<Value>> = records.iter().map(collect_process_ids).collect();
let format = output_format.unwrap_or("links-notation");
match format_record_list_with_current_times(&records, format, ¤t_times, &process_ids) {
Ok(output) => StatusQueryResult {
success: true,
output: Some(output),
error: None,
},
Err(e) => StatusQueryResult {
success: false,
output: None,
error: Some(e),
},
}
}
pub fn query_status(
store: Option<&ExecutionStore>,
identifier: &str,
output_format: Option<&str>,
) -> StatusQueryResult {
let store = match store {
Some(s) => s,
None => {
return StatusQueryResult {
success: false,
output: None,
error: Some("Execution tracking is disabled.".to_string()),
}
}
};
let record = match store.get(identifier) {
Some(r) => r,
None => {
return StatusQueryResult {
success: false,
output: None,
error: Some(format!(
"No execution found with UUID or session name: {}",
identifier
)),
}
}
};
let enriched = enrich_detached_status(&record);
let current_time = attach_current_time(&enriched);
let process_ids = collect_process_ids(&enriched);
let format = output_format.unwrap_or("links-notation");
match format_record_with_enrichments(
&enriched,
format,
current_time.as_deref(),
process_ids.as_ref(),
) {
Ok(output) => StatusQueryResult {
success: true,
output: Some(output),
error: None,
},
Err(e) => StatusQueryResult {
success: false,
output: None,
error: Some(e),
},
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::execution_store::ExecutionRecordOptions;
use serde_json::json;
fn executing_record() -> ExecutionRecord {
ExecutionRecord::with_options(ExecutionRecordOptions {
command: "sleep 60".to_string(),
uuid: Some("issue-126-rust".to_string()),
pid: Some(667105),
status: Some(ExecutionStatus::Executing),
log_path: Some("/tmp/issue-126.log".to_string()),
start_time: Some("2026-04-23T10:00:00Z".to_string()),
working_directory: Some("/home/user".to_string()),
shell: Some("/bin/bash".to_string()),
platform: Some("linux".to_string()),
..Default::default()
})
}
#[test]
fn links_notation_indents_nested_process_id_arrays() {
let process_ids = json!({
"wrapperPid": 667105,
"screenPid": 667120,
"commandPids": [667121, 667122],
});
let output = format_record_with_enrichments(
&executing_record(),
"links-notation",
Some("2026-04-23T10:10:13.042Z"),
Some(&process_ids),
)
.expect("links-notation should format");
assert!(
output.contains(
" commandPids\n (\n 667121\n 667122\n )"
),
"processIds should be a nested indented block, output: {}",
output
);
assert!(
!output.contains("\n(\n"),
"opening parenthesis must not start at column 1: {}",
output
);
}
}