1use crate::execution_attempt::{read_attempt_activity, read_attempt_log_tail};
9use crate::execution_control::collect_process_ids;
10use crate::execution_store::{ExecutionRecord, ExecutionStatus, ExecutionStore};
11use crate::exit_reason::{
12 describe_exit_code, resolve_exit_reason, resolve_memory_exhaustion, CGROUP_OOM_EXIT_REASON,
13};
14use crate::isolation::isolation_log::FATAL_MARKER_TAIL_BYTES;
15use crate::output_blocks::{escape_for_links_notation, format_value_for_links_notation};
16use crate::status_footer::{parse_footer_from_tail, read_footer_from_log};
17use crate::status_probe::{
18 apply_end_time, backend_exit_code, is_detached_docker_record, is_detached_session_alive,
19 read_docker_state, resolve_oom_observation,
20};
21use chrono::Utc;
22use serde_json::Value;
23
24pub fn read_exit_code_from_log(log_path: &str) -> Option<i32> {
30 read_footer_from_log(log_path).exit_code
31}
32
33pub fn enrich_detached_status(record: &ExecutionRecord) -> ExecutionRecord {
42 let mut enriched = resolve_detached_status(record);
43 if let Some(state) = crate::cpu_penalty_monitor::read_state(record) {
44 enriched.cpu_penalty = Some(state);
45 }
46 read_attempt_activity(&mut enriched);
47 if enriched.status != ExecutionStatus::Executed {
48 return enriched;
49 }
50 let tail = crate::exit_evidence::record_tail(
51 &enriched,
52 read_attempt_log_tail(&enriched, FATAL_MARKER_TAIL_BYTES),
53 );
54 let oom_kills = enriched.cgroup_memory.and_then(|memory| memory.oom_kills);
55 if enriched.exit_reason.is_none()
58 || enriched.exit_reason.as_deref() == Some(CGROUP_OOM_EXIT_REASON)
59 {
60 enriched.exit_reason = resolve_exit_reason(
61 enriched.exit_code,
62 tail.as_deref(),
63 enriched.oom_killed,
64 oom_kills,
65 );
66 }
67 if enriched
68 .memory_exhausted_reason
69 .as_deref()
70 .is_some_and(|s| s.starts_with("Docker reported") || s.starts_with("cgroup memory.events"))
71 {
72 enriched.memory_exhausted = None;
73 enriched.memory_exhausted_reason = None;
74 }
75 if enriched.memory_exhausted.is_none() {
76 if let Some(memory) = resolve_memory_exhaustion(
77 enriched.exit_code,
78 tail.as_deref(),
79 enriched.oom_killed,
80 oom_kills,
81 ) {
82 enriched.memory_exhausted = Some(memory.memory_exhausted);
83 enriched.memory_exhausted_reason = Some(memory.memory_exhausted_reason);
84 }
85 }
86 enriched
87}
88
89fn resolve_detached_status(record: &ExecutionRecord) -> ExecutionRecord {
94 if crate::launch_owner::has_active_launch(record) {
95 return record.clone();
96 }
97 let tail = read_attempt_log_tail(record, FATAL_MARKER_TAIL_BYTES);
98 let footer = parse_footer_from_tail(tail.as_deref().unwrap_or(""));
99 let footer_exit = footer.exit_code;
100 let is_detached_docker = is_detached_docker_record(record);
101 let docker_state = if is_detached_docker {
102 read_docker_state(record)
103 } else {
104 None
105 };
106
107 let oom_killed = resolve_oom_observation(record, docker_state.as_ref());
109 let clone_record = || {
110 let mut enriched = record.clone();
111 if oom_killed.is_some() {
112 enriched.oom_killed = oom_killed;
113 }
114 enriched
115 };
116
117 let alive = if is_detached_docker {
118 docker_state.as_ref().map(|state| state.running)
119 } else {
120 is_detached_session_alive(record)
121 };
122
123 let alive = match alive {
124 Some(value) => value,
125 None => {
126 let is_detached =
134 record.options.get("isolationMode").and_then(|v| v.as_str()) == Some("detached");
135 if is_detached && record.status == ExecutionStatus::Executing {
136 if footer_exit.is_some() {
137 let mut enriched = clone_record();
138 enriched.status = ExecutionStatus::Executed;
139 enriched.exit_code = footer_exit;
140 apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
141 return enriched;
142 }
143 if oom_killed == Some(true) {
144 let mut enriched = clone_record();
151 enriched.status = ExecutionStatus::Executed;
152 enriched.exit_code = Some(137);
153 apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
154 return enriched;
155 }
156 }
157 return clone_record();
158 }
159 };
160
161 let mut enriched = clone_record();
162
163 if alive && enriched.status == ExecutionStatus::Executed {
164 if enriched.exit_code.is_none() && footer_exit.is_none() {
171 enriched.status = ExecutionStatus::Executing;
173 enriched.exit_code = None;
174 enriched.end_time = None;
175 enriched.end_time_source = None;
176 }
177 } else if !alive && enriched.status == ExecutionStatus::Executing {
179 enriched.status = ExecutionStatus::Executed;
187 if enriched.exit_code.is_none() {
188 enriched.exit_code = Some(
189 backend_exit_code(docker_state.as_ref())
190 .or(footer_exit)
191 .or(if oom_killed == Some(true) {
192 Some(137)
193 } else {
194 None
195 })
196 .unwrap_or(-1),
197 );
198 }
199 apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
200 }
201
202 enriched
203}
204
205pub fn attach_current_time(record: &ExecutionRecord) -> Option<String> {
210 if record.status == ExecutionStatus::Executing {
211 Some(Utc::now().to_rfc3339())
212 } else {
213 None
214 }
215}
216
217pub fn format_record_as_links_notation(record: &ExecutionRecord) -> String {
229 format_record_as_links_notation_with_current_time(record, None)
230}
231
232pub fn format_record_as_links_notation_with_current_time(
235 record: &ExecutionRecord,
236 current_time: Option<&str>,
237) -> String {
238 format_record_as_links_notation_with_enrichments(record, current_time, None)
239}
240
241fn append_links_array(lines: &mut Vec<String>, values: &[Value], indent: usize) {
242 let prefix = " ".repeat(indent);
243 if values.is_empty() {
244 lines.push(format!("{}()", prefix));
245 return;
246 }
247
248 lines.push(format!("{}(", prefix));
249 for value in values {
250 match value {
251 Value::Array(nested) => append_links_array(lines, nested, indent + 2),
252 Value::Object(map) => {
253 for (child_key, child_value) in map {
254 if !child_value.is_null() {
255 append_links_value(lines, child_key, child_value, indent + 2);
256 }
257 }
258 }
259 _ => lines.push(format!(
260 "{}{}",
261 " ".repeat(indent + 2),
262 format_value_for_links_notation(value)
263 )),
264 }
265 }
266 lines.push(format!("{})", prefix));
267}
268
269fn append_links_value(lines: &mut Vec<String>, key: &str, value: &Value, indent: usize) {
270 let prefix = " ".repeat(indent);
271 match value {
272 Value::Object(map) => {
273 if map.is_empty() {
274 return;
275 }
276 lines.push(format!("{}{}", prefix, key));
277 for (child_key, child_value) in map {
278 if !child_value.is_null() {
279 append_links_value(lines, child_key, child_value, indent + 4);
280 }
281 }
282 }
283 Value::Array(values) => {
284 lines.push(format!("{}{}", prefix, key));
285 append_links_array(lines, values, indent + 2);
286 }
287 _ => lines.push(format!(
288 "{}{} {}",
289 prefix,
290 key,
291 format_value_for_links_notation(value)
292 )),
293 }
294}
295
296fn format_record_as_links_notation_with_enrichments(
297 record: &ExecutionRecord,
298 current_time: Option<&str>,
299 process_ids: Option<&Value>,
300) -> String {
301 let json = record.to_json();
302 let mut lines = vec![record.uuid.clone()];
303
304 if let Value::Object(map) = json {
305 for (key, value) in map {
306 if !value.is_null() {
307 if key == "options" {
308 if let Value::Object(opts) = &value {
310 if !opts.is_empty() {
311 lines.push(" options".to_string());
312 for (opt_key, opt_value) in opts {
313 if !opt_value.is_null() {
314 let formatted = format_value_for_links_notation(opt_value);
315 lines.push(format!(" {} {}", opt_key, formatted));
316 }
317 }
318 }
319 }
320 } else if value.is_object() {
321 append_links_value(&mut lines, &key, &value, 2);
324 } else {
325 let formatted_value = match &value {
326 Value::String(s) => escape_for_links_notation(s),
327 Value::Bool(b) => b.to_string(),
328 Value::Number(n) => n.to_string(),
329 Value::Null => "null".to_string(),
330 Value::Object(_) | Value::Array(_) => {
331 format_value_for_links_notation(&value)
333 }
334 };
335 lines.push(format!(" {} {}", key, formatted_value));
336 }
337 }
338
339 if key == "pid" {
342 if let Some(process_ids) = process_ids {
343 append_links_value(&mut lines, "processIds", process_ids, 2);
344 }
345 }
346
347 if key == "startTime" {
349 if let Some(ct) = current_time {
350 lines.push(format!(" currentTime {}", escape_for_links_notation(ct)));
351 }
352 }
353 }
354 }
355
356 lines.join("\n")
357}
358
359pub fn format_record_as_text(record: &ExecutionRecord) -> String {
361 format_record_as_text_with_current_time(record, None)
362}
363
364pub fn format_record_as_text_with_current_time(
367 record: &ExecutionRecord,
368 current_time: Option<&str>,
369) -> String {
370 format_record_as_text_with_enrichments(record, current_time, None)
371}
372
373fn append_text_process_ids(lines: &mut Vec<String>, process_ids: &Value) {
374 let Value::Object(map) = process_ids else {
375 return;
376 };
377 if map.is_empty() {
378 return;
379 }
380
381 lines.push("Process IDs:".to_string());
382 for (key, value) in map {
383 let value_str = match value {
384 Value::String(s) => s.clone(),
385 Value::Bool(b) => b.to_string(),
386 Value::Number(n) => n.to_string(),
387 Value::Null => "null".to_string(),
388 other => serde_json::to_string(other).unwrap_or_default(),
389 };
390 lines.push(format!(" {}: {}", key, value_str));
391 }
392}
393
394fn format_record_as_text_with_enrichments(
395 record: &ExecutionRecord,
396 current_time: Option<&str>,
397 process_ids: Option<&Value>,
398) -> String {
399 let exit_code_str = record
400 .exit_code
401 .map(|c| describe_exit_code(Some(c)).text)
405 .unwrap_or_else(|| "N/A".to_string());
406 let pid_str = record
407 .pid
408 .map(|p| p.to_string())
409 .unwrap_or_else(|| "N/A".to_string());
410 let end_time_str = record.end_time.as_deref().unwrap_or("N/A");
411
412 let mut lines = vec![
413 "Execution Status".to_string(),
414 "=".repeat(50),
415 format!("UUID: {}", record.uuid),
416 format!("Status: {}", record.status),
417 format!("Command: {}", record.command),
418 format!("Exit Code: {}", exit_code_str),
419 ];
420 if let Some(oom_killed) = record.oom_killed {
421 lines.push(format!("OOM Killed: {}", oom_killed));
422 }
423 if let Some(ref exit_reason) = record.exit_reason {
424 lines.push(format!("Exit Reason: {}", exit_reason));
425 }
426 if let Some(memory_exhausted) = record.memory_exhausted {
427 lines.push(format!("Memory Exhausted: {}", memory_exhausted));
428 }
429 if let Some(ref reason) = record.memory_exhausted_reason {
430 lines.push(format!("Memory Evidence: {}", reason));
431 }
432 if let Some(ref memory) = record.cgroup_memory {
434 lines.push(format!(
435 "Cgroup Memory: {}",
436 crate::cgroup_memory::format_cgroup_memory(memory)
437 ));
438 }
439 if let Some(ref state) = record.cpu_penalty {
440 lines.push(format!("CPU Penalty: {}", state));
441 }
442 lines.push(format!("PID: {}", pid_str));
443 if let Some(process_ids) = process_ids {
444 append_text_process_ids(&mut lines, process_ids);
445 }
446 lines.extend([
447 format!("Working Directory: {}", record.working_directory),
448 format!("Shell: {}", record.shell),
449 format!("Platform: {}", record.platform),
450 format!("Start Time: {}", record.start_time),
451 ]);
452 if let Some(ct) = current_time {
453 lines.push(format!("Current Time: {}", ct));
454 }
455 lines.push(format!("End Time: {}", end_time_str));
456 if let Some(ref source) = record.end_time_source {
457 lines.push(format!("End Time Source: {}", source));
460 }
461 if let Some(ref started) = record.container_started_at {
462 lines.push(format!("Container Started: {}", started));
463 }
464 if let Some(ref detected) = record.stale_detected_at {
465 lines.push(format!("Stale Detected At: {}", detected));
466 }
467 lines.push(format!("Log Path: {}", record.log_path));
468 if let Some(attempt) = &record.attempt {
469 lines.extend([
470 format!("Attempt: {}", attempt.number),
471 format!("Attempt Started: {}", attempt.started_at),
472 format!(
473 "Attempt Log Offset: {}",
474 attempt
475 .log_offset
476 .map(|offset| offset.to_string())
477 .unwrap_or_else(|| "N/A".into())
478 ),
479 format!(
480 "Launch Accepted: {}",
481 attempt.launch_accepted_at.as_deref().unwrap_or("N/A")
482 ),
483 format!(
484 "Watcher Attached: {}",
485 attempt.watcher_attached_at.as_deref().unwrap_or("N/A")
486 ),
487 format!(
488 "Last Output: {}",
489 attempt.last_output_at.as_deref().unwrap_or("N/A")
490 ),
491 ]);
492 }
493
494 if !record.options.is_empty() {
496 lines.push("Options:".to_string());
497 for (key, value) in &record.options {
498 let value_str = match value {
499 Value::String(s) => s.clone(),
500 Value::Bool(b) => b.to_string(),
501 Value::Number(n) => n.to_string(),
502 Value::Null => "null".to_string(),
503 other => serde_json::to_string(other).unwrap_or_default(),
504 };
505 lines.push(format!(" {}: {}", key, value_str));
506 }
507 }
508
509 lines.join("\n")
510}
511
512fn record_json_with_enrichments(
513 record: &ExecutionRecord,
514 current_time: Option<&str>,
515 process_ids: Option<&Value>,
516) -> Value {
517 let mut json = record.to_json();
518 if let Value::Object(map) = &mut json {
519 if let Some(process_ids) = process_ids {
520 map.insert("processIds".to_string(), process_ids.clone());
521 }
522 if let Some(ct) = current_time {
523 map.insert("currentTime".to_string(), Value::String(ct.to_string()));
524 }
525 }
526 json
527}
528
529pub fn format_record(record: &ExecutionRecord, format: &str) -> Result<String, String> {
531 format_record_with_current_time(record, format, None)
532}
533
534pub fn format_record_with_current_time(
538 record: &ExecutionRecord,
539 format: &str,
540 current_time: Option<&str>,
541) -> Result<String, String> {
542 format_record_with_enrichments(record, format, current_time, None)
543}
544
545fn format_record_with_enrichments(
546 record: &ExecutionRecord,
547 format: &str,
548 current_time: Option<&str>,
549 process_ids: Option<&Value>,
550) -> Result<String, String> {
551 match format {
552 "links-notation" => Ok(format_record_as_links_notation_with_enrichments(
553 record,
554 current_time,
555 process_ids,
556 )),
557 "json" => serde_json::to_string_pretty(&record_json_with_enrichments(
558 record,
559 current_time,
560 process_ids,
561 ))
562 .map_err(|e| format!("Failed to serialize to JSON: {}", e)),
563 "text" => Ok(format_record_as_text_with_enrichments(
564 record,
565 current_time,
566 process_ids,
567 )),
568 _ => Err(format!("Unknown output format: {}", format)),
569 }
570}
571
572fn sort_records_by_start_time_desc(records: &mut [ExecutionRecord]) {
573 records.sort_by(|a, b| b.start_time.cmp(&a.start_time));
574}
575
576fn indent_block(block: &str, spaces: usize) -> String {
577 let prefix = " ".repeat(spaces);
578 block
579 .lines()
580 .map(|line| format!("{}{}", prefix, line))
581 .collect::<Vec<_>>()
582 .join("\n")
583}
584
585pub fn format_record_list_as_links_notation(records: &[ExecutionRecord]) -> String {
587 let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
588 let process_ids = vec![None; records.len()];
589 format_record_list_as_links_notation_with_current_times(records, ¤t_times, &process_ids)
590}
591
592fn format_record_list_as_links_notation_with_current_times(
593 records: &[ExecutionRecord],
594 current_times: &[Option<String>],
595 process_ids: &[Option<Value>],
596) -> String {
597 let mut lines = vec![
598 "executions".to_string(),
599 format!(" count {}", records.len()),
600 ];
601
602 if records.is_empty() {
603 lines.push(" records ()".to_string());
604 return lines.join("\n");
605 }
606
607 lines.push(" records".to_string());
608 for ((record, current_time), process_ids) in records
609 .iter()
610 .zip(current_times.iter())
611 .zip(process_ids.iter())
612 {
613 let block = format_record_as_links_notation_with_enrichments(
614 record,
615 current_time.as_deref(),
616 process_ids.as_ref(),
617 );
618 lines.push(indent_block(&block, 4));
619 }
620
621 lines.join("\n")
622}
623
624pub fn format_record_list_as_text(records: &[ExecutionRecord]) -> String {
626 let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
627 let process_ids = vec![None; records.len()];
628 format_record_list_as_text_with_current_times(records, ¤t_times, &process_ids)
629}
630
631fn format_record_list_as_text_with_current_times(
632 records: &[ExecutionRecord],
633 current_times: &[Option<String>],
634 process_ids: &[Option<Value>],
635) -> String {
636 let mut lines = vec![
637 "Executions".to_string(),
638 "=".repeat(50),
639 format!("Count: {}", records.len()),
640 ];
641
642 for ((record, current_time), process_ids) in records
643 .iter()
644 .zip(current_times.iter())
645 .zip(process_ids.iter())
646 {
647 lines.push(String::new());
648 lines.push(format_record_as_text_with_enrichments(
649 record,
650 current_time.as_deref(),
651 process_ids.as_ref(),
652 ));
653 }
654
655 lines.join("\n")
656}
657
658fn record_list_json_with_current_times(
659 records: &[ExecutionRecord],
660 current_times: &[Option<String>],
661 process_ids: &[Option<Value>],
662) -> Value {
663 let executions: Vec<Value> = records
664 .iter()
665 .zip(current_times.iter())
666 .zip(process_ids.iter())
667 .map(|((record, current_time), process_ids)| {
668 record_json_with_enrichments(record, current_time.as_deref(), process_ids.as_ref())
669 })
670 .collect();
671
672 serde_json::json!({
673 "count": records.len(),
674 "executions": executions,
675 })
676}
677
678pub fn format_record_list(records: &[ExecutionRecord], format: &str) -> Result<String, String> {
680 let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
681 let process_ids = vec![None; records.len()];
682 format_record_list_with_current_times(records, format, ¤t_times, &process_ids)
683}
684
685fn format_record_list_with_current_times(
686 records: &[ExecutionRecord],
687 format: &str,
688 current_times: &[Option<String>],
689 process_ids: &[Option<Value>],
690) -> Result<String, String> {
691 match format {
692 "links-notation" => Ok(format_record_list_as_links_notation_with_current_times(
693 records,
694 current_times,
695 process_ids,
696 )),
697 "json" => serde_json::to_string_pretty(&record_list_json_with_current_times(
698 records,
699 current_times,
700 process_ids,
701 ))
702 .map_err(|e| format!("Failed to serialize to JSON: {}", e)),
703 "text" => Ok(format_record_list_as_text_with_current_times(
704 records,
705 current_times,
706 process_ids,
707 )),
708 _ => Err(format!("Unknown output format: {}", format)),
709 }
710}
711
712pub struct StatusQueryResult {
714 pub success: bool,
715 pub output: Option<String>,
716 pub error: Option<String>,
717}
718
719pub fn list_executions(
721 store: Option<&ExecutionStore>,
722 output_format: Option<&str>,
723) -> StatusQueryResult {
724 list_executions_filtered(store, output_format, false)
725}
726
727pub fn list_executions_filtered(
733 store: Option<&ExecutionStore>,
734 output_format: Option<&str>,
735 running_only: bool,
736) -> StatusQueryResult {
737 let store = match store {
738 Some(s) => s,
739 None => {
740 return StatusQueryResult {
741 success: false,
742 output: None,
743 error: Some("Execution tracking is disabled.".to_string()),
744 }
745 }
746 };
747
748 let mut records: Vec<ExecutionRecord> =
749 store.get_all().iter().map(enrich_detached_status).collect();
750 if running_only {
751 records.retain(|record| record.status == ExecutionStatus::Executing);
752 }
753 sort_records_by_start_time_desc(&mut records);
754 let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
755 let process_ids: Vec<Option<Value>> = records.iter().map(collect_process_ids).collect();
756 let format = output_format.unwrap_or("links-notation");
757
758 match format_record_list_with_current_times(&records, format, ¤t_times, &process_ids) {
759 Ok(output) => StatusQueryResult {
760 success: true,
761 output: Some(output),
762 error: None,
763 },
764 Err(e) => StatusQueryResult {
765 success: false,
766 output: None,
767 error: Some(e),
768 },
769 }
770}
771
772pub fn query_status(
774 store: Option<&ExecutionStore>,
775 identifier: &str,
776 output_format: Option<&str>,
777) -> StatusQueryResult {
778 let store = match store {
779 Some(s) => s,
780 None => {
781 return StatusQueryResult {
782 success: false,
783 output: None,
784 error: Some("Execution tracking is disabled.".to_string()),
785 }
786 }
787 };
788
789 let record = match store.get(identifier) {
790 Some(r) => r,
791 None => {
792 return StatusQueryResult {
793 success: false,
794 output: None,
795 error: Some(format!(
796 "No execution found with UUID or session name: {}",
797 identifier
798 )),
799 }
800 }
801 };
802
803 let enriched = enrich_detached_status(&record);
805 let current_time = attach_current_time(&enriched);
807 let process_ids = collect_process_ids(&enriched);
808
809 let format = output_format.unwrap_or("links-notation");
810 match format_record_with_enrichments(
811 &enriched,
812 format,
813 current_time.as_deref(),
814 process_ids.as_ref(),
815 ) {
816 Ok(output) => StatusQueryResult {
817 success: true,
818 output: Some(output),
819 error: None,
820 },
821 Err(e) => StatusQueryResult {
822 success: false,
823 output: None,
824 error: Some(e),
825 },
826 }
827}
828
829#[cfg(test)]
830mod tests {
831 use super::*;
832 use crate::execution_store::ExecutionRecordOptions;
833 use serde_json::json;
834
835 fn executing_record() -> ExecutionRecord {
836 ExecutionRecord::with_options(ExecutionRecordOptions {
837 command: "sleep 60".to_string(),
838 uuid: Some("issue-126-rust".to_string()),
839 pid: Some(667105),
840 status: Some(ExecutionStatus::Executing),
841 log_path: Some("/tmp/issue-126.log".to_string()),
842 start_time: Some("2026-04-23T10:00:00Z".to_string()),
843 working_directory: Some("/home/user".to_string()),
844 shell: Some("/bin/bash".to_string()),
845 platform: Some("linux".to_string()),
846 ..Default::default()
847 })
848 }
849
850 #[test]
851 fn links_notation_indents_nested_process_id_arrays() {
852 let process_ids = json!({
853 "wrapperPid": 667105,
854 "screenPid": 667120,
855 "commandPids": [667121, 667122],
856 });
857 let output = format_record_with_enrichments(
858 &executing_record(),
859 "links-notation",
860 Some("2026-04-23T10:10:13.042Z"),
861 Some(&process_ids),
862 )
863 .expect("links-notation should format");
864
865 assert!(
866 output.contains(
867 " commandPids\n (\n 667121\n 667122\n )"
868 ),
869 "processIds should be a nested indented block, output: {}",
870 output
871 );
872 assert!(
873 !output.contains("\n(\n"),
874 "opening parenthesis must not start at column 1: {}",
875 output
876 );
877 }
878}