1use crate::execution_attempt::{read_attempt_activity, read_attempt_log_tail};
9use crate::execution_control::collect_process_ids;
10use crate::execution_store::{ExecutionRecord, ExecutionStatus, ExecutionStore};
11use crate::exit_reason::{
12 describe_exit_code, resolve_exit_reason, resolve_memory_exhaustion, CGROUP_OOM_EXIT_REASON,
13};
14use crate::isolation::isolation_log::FATAL_MARKER_TAIL_BYTES;
15use crate::output_blocks::{escape_for_links_notation, format_value_for_links_notation};
16use crate::status_footer::{parse_footer_from_tail, read_footer_from_log};
17use crate::status_probe::{
18 apply_end_time, backend_exit_code, is_detached_docker_record, is_detached_session_alive,
19 read_docker_state, resolve_oom_observation,
20};
21use chrono::Utc;
22use serde_json::Value;
23
24pub fn read_exit_code_from_log(log_path: &str) -> Option<i32> {
30 read_footer_from_log(log_path).exit_code
31}
32
33pub fn enrich_detached_status(record: &ExecutionRecord) -> ExecutionRecord {
42 let mut enriched = resolve_detached_status(record);
43 read_attempt_activity(&mut enriched);
44 if enriched.status != ExecutionStatus::Executed {
45 return enriched;
46 }
47 let tail = read_attempt_log_tail(&enriched, FATAL_MARKER_TAIL_BYTES);
48 let oom_kills = enriched.cgroup_memory.and_then(|memory| memory.oom_kills);
49 if enriched.exit_reason.is_none()
52 || enriched.exit_reason.as_deref() == Some(CGROUP_OOM_EXIT_REASON)
53 {
54 enriched.exit_reason = resolve_exit_reason(
55 enriched.exit_code,
56 tail.as_deref(),
57 enriched.oom_killed,
58 oom_kills,
59 );
60 }
61 if enriched.memory_exhausted.is_none() {
62 if let Some(memory) = resolve_memory_exhaustion(
63 enriched.exit_code,
64 tail.as_deref(),
65 enriched.oom_killed,
66 oom_kills,
67 ) {
68 enriched.memory_exhausted = Some(memory.memory_exhausted);
69 enriched.memory_exhausted_reason = Some(memory.memory_exhausted_reason);
70 }
71 }
72 enriched
73}
74
75fn resolve_detached_status(record: &ExecutionRecord) -> ExecutionRecord {
80 let tail = read_attempt_log_tail(record, FATAL_MARKER_TAIL_BYTES);
81 let footer = parse_footer_from_tail(tail.as_deref().unwrap_or(""));
82 let footer_exit = footer.exit_code;
83 let is_detached_docker = is_detached_docker_record(record);
84 let docker_state = if is_detached_docker {
85 read_docker_state(record)
86 } else {
87 None
88 };
89
90 let oom_killed = resolve_oom_observation(record, docker_state.as_ref());
92 let clone_record = || {
93 let mut enriched = record.clone();
94 if oom_killed.is_some() {
95 enriched.oom_killed = oom_killed;
96 }
97 enriched
98 };
99
100 let alive = if is_detached_docker {
101 docker_state.as_ref().map(|state| state.running)
102 } else {
103 is_detached_session_alive(record)
104 };
105
106 let alive = match alive {
107 Some(value) => value,
108 None => {
109 let is_detached =
117 record.options.get("isolationMode").and_then(|v| v.as_str()) == Some("detached");
118 if is_detached && record.status == ExecutionStatus::Executing {
119 if footer_exit.is_some() {
120 let mut enriched = clone_record();
121 enriched.status = ExecutionStatus::Executed;
122 enriched.exit_code = footer_exit;
123 apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
124 return enriched;
125 }
126 if oom_killed == Some(true) {
127 let mut enriched = clone_record();
134 enriched.status = ExecutionStatus::Executed;
135 enriched.exit_code = Some(137);
136 apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
137 return enriched;
138 }
139 }
140 return clone_record();
141 }
142 };
143
144 let mut enriched = clone_record();
145
146 if alive && enriched.status == ExecutionStatus::Executed {
147 if enriched.exit_code.is_none() && footer_exit.is_none() {
154 enriched.status = ExecutionStatus::Executing;
156 enriched.exit_code = None;
157 enriched.end_time = None;
158 enriched.end_time_source = None;
159 }
160 } else if !alive && enriched.status == ExecutionStatus::Executing {
162 enriched.status = ExecutionStatus::Executed;
170 if enriched.exit_code.is_none() {
171 enriched.exit_code = Some(
172 backend_exit_code(docker_state.as_ref())
173 .or(footer_exit)
174 .or(if oom_killed == Some(true) {
175 Some(137)
176 } else {
177 None
178 })
179 .unwrap_or(-1),
180 );
181 }
182 apply_end_time(&mut enriched, docker_state.as_ref(), &footer);
183 }
184
185 enriched
186}
187
188pub fn attach_current_time(record: &ExecutionRecord) -> Option<String> {
193 if record.status == ExecutionStatus::Executing {
194 Some(Utc::now().to_rfc3339())
195 } else {
196 None
197 }
198}
199
200pub fn format_record_as_links_notation(record: &ExecutionRecord) -> String {
212 format_record_as_links_notation_with_current_time(record, None)
213}
214
215pub fn format_record_as_links_notation_with_current_time(
218 record: &ExecutionRecord,
219 current_time: Option<&str>,
220) -> String {
221 format_record_as_links_notation_with_enrichments(record, current_time, None)
222}
223
224fn append_links_array(lines: &mut Vec<String>, values: &[Value], indent: usize) {
225 let prefix = " ".repeat(indent);
226 if values.is_empty() {
227 lines.push(format!("{}()", prefix));
228 return;
229 }
230
231 lines.push(format!("{}(", prefix));
232 for value in values {
233 match value {
234 Value::Array(nested) => append_links_array(lines, nested, indent + 2),
235 Value::Object(map) => {
236 for (child_key, child_value) in map {
237 if !child_value.is_null() {
238 append_links_value(lines, child_key, child_value, indent + 2);
239 }
240 }
241 }
242 _ => lines.push(format!(
243 "{}{}",
244 " ".repeat(indent + 2),
245 format_value_for_links_notation(value)
246 )),
247 }
248 }
249 lines.push(format!("{})", prefix));
250}
251
252fn append_links_value(lines: &mut Vec<String>, key: &str, value: &Value, indent: usize) {
253 let prefix = " ".repeat(indent);
254 match value {
255 Value::Object(map) => {
256 if map.is_empty() {
257 return;
258 }
259 lines.push(format!("{}{}", prefix, key));
260 for (child_key, child_value) in map {
261 if !child_value.is_null() {
262 append_links_value(lines, child_key, child_value, indent + 4);
263 }
264 }
265 }
266 Value::Array(values) => {
267 lines.push(format!("{}{}", prefix, key));
268 append_links_array(lines, values, indent + 2);
269 }
270 _ => lines.push(format!(
271 "{}{} {}",
272 prefix,
273 key,
274 format_value_for_links_notation(value)
275 )),
276 }
277}
278
279fn format_record_as_links_notation_with_enrichments(
280 record: &ExecutionRecord,
281 current_time: Option<&str>,
282 process_ids: Option<&Value>,
283) -> String {
284 let json = record.to_json();
285 let mut lines = vec![record.uuid.clone()];
286
287 if let Value::Object(map) = json {
288 for (key, value) in map {
289 if !value.is_null() {
290 if key == "options" {
291 if let Value::Object(opts) = &value {
293 if !opts.is_empty() {
294 lines.push(" options".to_string());
295 for (opt_key, opt_value) in opts {
296 if !opt_value.is_null() {
297 let formatted = format_value_for_links_notation(opt_value);
298 lines.push(format!(" {} {}", opt_key, formatted));
299 }
300 }
301 }
302 }
303 } else if value.is_object() {
304 append_links_value(&mut lines, &key, &value, 2);
307 } else {
308 let formatted_value = match &value {
309 Value::String(s) => escape_for_links_notation(s),
310 Value::Bool(b) => b.to_string(),
311 Value::Number(n) => n.to_string(),
312 Value::Null => "null".to_string(),
313 Value::Object(_) | Value::Array(_) => {
314 format_value_for_links_notation(&value)
316 }
317 };
318 lines.push(format!(" {} {}", key, formatted_value));
319 }
320 }
321
322 if key == "pid" {
325 if let Some(process_ids) = process_ids {
326 append_links_value(&mut lines, "processIds", process_ids, 2);
327 }
328 }
329
330 if key == "startTime" {
332 if let Some(ct) = current_time {
333 lines.push(format!(" currentTime {}", escape_for_links_notation(ct)));
334 }
335 }
336 }
337 }
338
339 lines.join("\n")
340}
341
342pub fn format_record_as_text(record: &ExecutionRecord) -> String {
344 format_record_as_text_with_current_time(record, None)
345}
346
347pub fn format_record_as_text_with_current_time(
350 record: &ExecutionRecord,
351 current_time: Option<&str>,
352) -> String {
353 format_record_as_text_with_enrichments(record, current_time, None)
354}
355
356fn append_text_process_ids(lines: &mut Vec<String>, process_ids: &Value) {
357 let Value::Object(map) = process_ids else {
358 return;
359 };
360 if map.is_empty() {
361 return;
362 }
363
364 lines.push("Process IDs:".to_string());
365 for (key, value) in map {
366 let value_str = match value {
367 Value::String(s) => s.clone(),
368 Value::Bool(b) => b.to_string(),
369 Value::Number(n) => n.to_string(),
370 Value::Null => "null".to_string(),
371 other => serde_json::to_string(other).unwrap_or_default(),
372 };
373 lines.push(format!(" {}: {}", key, value_str));
374 }
375}
376
377fn format_record_as_text_with_enrichments(
378 record: &ExecutionRecord,
379 current_time: Option<&str>,
380 process_ids: Option<&Value>,
381) -> String {
382 let exit_code_str = record
383 .exit_code
384 .map(|c| describe_exit_code(Some(c)).text)
388 .unwrap_or_else(|| "N/A".to_string());
389 let pid_str = record
390 .pid
391 .map(|p| p.to_string())
392 .unwrap_or_else(|| "N/A".to_string());
393 let end_time_str = record.end_time.as_deref().unwrap_or("N/A");
394
395 let mut lines = vec![
396 "Execution Status".to_string(),
397 "=".repeat(50),
398 format!("UUID: {}", record.uuid),
399 format!("Status: {}", record.status),
400 format!("Command: {}", record.command),
401 format!("Exit Code: {}", exit_code_str),
402 ];
403 if let Some(oom_killed) = record.oom_killed {
404 lines.push(format!("OOM Killed: {}", oom_killed));
405 }
406 if let Some(ref exit_reason) = record.exit_reason {
407 lines.push(format!("Exit Reason: {}", exit_reason));
408 }
409 if let Some(memory_exhausted) = record.memory_exhausted {
410 lines.push(format!("Memory Exhausted: {}", memory_exhausted));
411 }
412 if let Some(ref reason) = record.memory_exhausted_reason {
413 lines.push(format!("Memory Evidence: {}", reason));
414 }
415 if let Some(ref memory) = record.cgroup_memory {
417 lines.push(format!(
418 "Cgroup Memory: {}",
419 crate::cgroup_memory::format_cgroup_memory(memory)
420 ));
421 }
422 lines.push(format!("PID: {}", pid_str));
423 if let Some(process_ids) = process_ids {
424 append_text_process_ids(&mut lines, process_ids);
425 }
426 lines.extend([
427 format!("Working Directory: {}", record.working_directory),
428 format!("Shell: {}", record.shell),
429 format!("Platform: {}", record.platform),
430 format!("Start Time: {}", record.start_time),
431 ]);
432 if let Some(ct) = current_time {
433 lines.push(format!("Current Time: {}", ct));
434 }
435 lines.push(format!("End Time: {}", end_time_str));
436 if let Some(ref source) = record.end_time_source {
437 lines.push(format!("End Time Source: {}", source));
440 }
441 if let Some(ref started) = record.container_started_at {
442 lines.push(format!("Container Started: {}", started));
443 }
444 if let Some(ref detected) = record.stale_detected_at {
445 lines.push(format!("Stale Detected At: {}", detected));
446 }
447 lines.push(format!("Log Path: {}", record.log_path));
448 if let Some(attempt) = &record.attempt {
449 lines.extend([
450 format!("Attempt: {}", attempt.number),
451 format!("Attempt Started: {}", attempt.started_at),
452 format!(
453 "Attempt Log Offset: {}",
454 attempt
455 .log_offset
456 .map(|offset| offset.to_string())
457 .unwrap_or_else(|| "N/A".into())
458 ),
459 format!(
460 "Launch Accepted: {}",
461 attempt.launch_accepted_at.as_deref().unwrap_or("N/A")
462 ),
463 format!(
464 "Watcher Attached: {}",
465 attempt.watcher_attached_at.as_deref().unwrap_or("N/A")
466 ),
467 format!(
468 "Last Output: {}",
469 attempt.last_output_at.as_deref().unwrap_or("N/A")
470 ),
471 ]);
472 }
473
474 if !record.options.is_empty() {
476 lines.push("Options:".to_string());
477 for (key, value) in &record.options {
478 let value_str = match value {
479 Value::String(s) => s.clone(),
480 Value::Bool(b) => b.to_string(),
481 Value::Number(n) => n.to_string(),
482 Value::Null => "null".to_string(),
483 other => serde_json::to_string(other).unwrap_or_default(),
484 };
485 lines.push(format!(" {}: {}", key, value_str));
486 }
487 }
488
489 lines.join("\n")
490}
491
492fn record_json_with_enrichments(
493 record: &ExecutionRecord,
494 current_time: Option<&str>,
495 process_ids: Option<&Value>,
496) -> Value {
497 let mut json = record.to_json();
498 if let Value::Object(map) = &mut json {
499 if let Some(process_ids) = process_ids {
500 map.insert("processIds".to_string(), process_ids.clone());
501 }
502 if let Some(ct) = current_time {
503 map.insert("currentTime".to_string(), Value::String(ct.to_string()));
504 }
505 }
506 json
507}
508
509pub fn format_record(record: &ExecutionRecord, format: &str) -> Result<String, String> {
511 format_record_with_current_time(record, format, None)
512}
513
514pub fn format_record_with_current_time(
518 record: &ExecutionRecord,
519 format: &str,
520 current_time: Option<&str>,
521) -> Result<String, String> {
522 format_record_with_enrichments(record, format, current_time, None)
523}
524
525fn format_record_with_enrichments(
526 record: &ExecutionRecord,
527 format: &str,
528 current_time: Option<&str>,
529 process_ids: Option<&Value>,
530) -> Result<String, String> {
531 match format {
532 "links-notation" => Ok(format_record_as_links_notation_with_enrichments(
533 record,
534 current_time,
535 process_ids,
536 )),
537 "json" => serde_json::to_string_pretty(&record_json_with_enrichments(
538 record,
539 current_time,
540 process_ids,
541 ))
542 .map_err(|e| format!("Failed to serialize to JSON: {}", e)),
543 "text" => Ok(format_record_as_text_with_enrichments(
544 record,
545 current_time,
546 process_ids,
547 )),
548 _ => Err(format!("Unknown output format: {}", format)),
549 }
550}
551
552fn sort_records_by_start_time_desc(records: &mut [ExecutionRecord]) {
553 records.sort_by(|a, b| b.start_time.cmp(&a.start_time));
554}
555
556fn indent_block(block: &str, spaces: usize) -> String {
557 let prefix = " ".repeat(spaces);
558 block
559 .lines()
560 .map(|line| format!("{}{}", prefix, line))
561 .collect::<Vec<_>>()
562 .join("\n")
563}
564
565pub fn format_record_list_as_links_notation(records: &[ExecutionRecord]) -> String {
567 let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
568 let process_ids = vec![None; records.len()];
569 format_record_list_as_links_notation_with_current_times(records, ¤t_times, &process_ids)
570}
571
572fn format_record_list_as_links_notation_with_current_times(
573 records: &[ExecutionRecord],
574 current_times: &[Option<String>],
575 process_ids: &[Option<Value>],
576) -> String {
577 let mut lines = vec![
578 "executions".to_string(),
579 format!(" count {}", records.len()),
580 ];
581
582 if records.is_empty() {
583 lines.push(" records ()".to_string());
584 return lines.join("\n");
585 }
586
587 lines.push(" records".to_string());
588 for ((record, current_time), process_ids) in records
589 .iter()
590 .zip(current_times.iter())
591 .zip(process_ids.iter())
592 {
593 let block = format_record_as_links_notation_with_enrichments(
594 record,
595 current_time.as_deref(),
596 process_ids.as_ref(),
597 );
598 lines.push(indent_block(&block, 4));
599 }
600
601 lines.join("\n")
602}
603
604pub fn format_record_list_as_text(records: &[ExecutionRecord]) -> String {
606 let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
607 let process_ids = vec![None; records.len()];
608 format_record_list_as_text_with_current_times(records, ¤t_times, &process_ids)
609}
610
611fn format_record_list_as_text_with_current_times(
612 records: &[ExecutionRecord],
613 current_times: &[Option<String>],
614 process_ids: &[Option<Value>],
615) -> String {
616 let mut lines = vec![
617 "Executions".to_string(),
618 "=".repeat(50),
619 format!("Count: {}", records.len()),
620 ];
621
622 for ((record, current_time), process_ids) in records
623 .iter()
624 .zip(current_times.iter())
625 .zip(process_ids.iter())
626 {
627 lines.push(String::new());
628 lines.push(format_record_as_text_with_enrichments(
629 record,
630 current_time.as_deref(),
631 process_ids.as_ref(),
632 ));
633 }
634
635 lines.join("\n")
636}
637
638fn record_list_json_with_current_times(
639 records: &[ExecutionRecord],
640 current_times: &[Option<String>],
641 process_ids: &[Option<Value>],
642) -> Value {
643 let executions: Vec<Value> = records
644 .iter()
645 .zip(current_times.iter())
646 .zip(process_ids.iter())
647 .map(|((record, current_time), process_ids)| {
648 record_json_with_enrichments(record, current_time.as_deref(), process_ids.as_ref())
649 })
650 .collect();
651
652 serde_json::json!({
653 "count": records.len(),
654 "executions": executions,
655 })
656}
657
658pub fn format_record_list(records: &[ExecutionRecord], format: &str) -> Result<String, String> {
660 let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
661 let process_ids = vec![None; records.len()];
662 format_record_list_with_current_times(records, format, ¤t_times, &process_ids)
663}
664
665fn format_record_list_with_current_times(
666 records: &[ExecutionRecord],
667 format: &str,
668 current_times: &[Option<String>],
669 process_ids: &[Option<Value>],
670) -> Result<String, String> {
671 match format {
672 "links-notation" => Ok(format_record_list_as_links_notation_with_current_times(
673 records,
674 current_times,
675 process_ids,
676 )),
677 "json" => serde_json::to_string_pretty(&record_list_json_with_current_times(
678 records,
679 current_times,
680 process_ids,
681 ))
682 .map_err(|e| format!("Failed to serialize to JSON: {}", e)),
683 "text" => Ok(format_record_list_as_text_with_current_times(
684 records,
685 current_times,
686 process_ids,
687 )),
688 _ => Err(format!("Unknown output format: {}", format)),
689 }
690}
691
692pub struct StatusQueryResult {
694 pub success: bool,
695 pub output: Option<String>,
696 pub error: Option<String>,
697}
698
699pub fn list_executions(
701 store: Option<&ExecutionStore>,
702 output_format: Option<&str>,
703) -> StatusQueryResult {
704 list_executions_filtered(store, output_format, false)
705}
706
707pub fn list_executions_filtered(
713 store: Option<&ExecutionStore>,
714 output_format: Option<&str>,
715 running_only: bool,
716) -> StatusQueryResult {
717 let store = match store {
718 Some(s) => s,
719 None => {
720 return StatusQueryResult {
721 success: false,
722 output: None,
723 error: Some("Execution tracking is disabled.".to_string()),
724 }
725 }
726 };
727
728 let mut records: Vec<ExecutionRecord> =
729 store.get_all().iter().map(enrich_detached_status).collect();
730 if running_only {
731 records.retain(|record| record.status == ExecutionStatus::Executing);
732 }
733 sort_records_by_start_time_desc(&mut records);
734 let current_times: Vec<Option<String>> = records.iter().map(attach_current_time).collect();
735 let process_ids: Vec<Option<Value>> = records.iter().map(collect_process_ids).collect();
736 let format = output_format.unwrap_or("links-notation");
737
738 match format_record_list_with_current_times(&records, format, ¤t_times, &process_ids) {
739 Ok(output) => StatusQueryResult {
740 success: true,
741 output: Some(output),
742 error: None,
743 },
744 Err(e) => StatusQueryResult {
745 success: false,
746 output: None,
747 error: Some(e),
748 },
749 }
750}
751
752pub fn query_status(
754 store: Option<&ExecutionStore>,
755 identifier: &str,
756 output_format: Option<&str>,
757) -> StatusQueryResult {
758 let store = match store {
759 Some(s) => s,
760 None => {
761 return StatusQueryResult {
762 success: false,
763 output: None,
764 error: Some("Execution tracking is disabled.".to_string()),
765 }
766 }
767 };
768
769 let record = match store.get(identifier) {
770 Some(r) => r,
771 None => {
772 return StatusQueryResult {
773 success: false,
774 output: None,
775 error: Some(format!(
776 "No execution found with UUID or session name: {}",
777 identifier
778 )),
779 }
780 }
781 };
782
783 let enriched = enrich_detached_status(&record);
785 let current_time = attach_current_time(&enriched);
787 let process_ids = collect_process_ids(&enriched);
788
789 let format = output_format.unwrap_or("links-notation");
790 match format_record_with_enrichments(
791 &enriched,
792 format,
793 current_time.as_deref(),
794 process_ids.as_ref(),
795 ) {
796 Ok(output) => StatusQueryResult {
797 success: true,
798 output: Some(output),
799 error: None,
800 },
801 Err(e) => StatusQueryResult {
802 success: false,
803 output: None,
804 error: Some(e),
805 },
806 }
807}
808
809#[cfg(test)]
810mod tests {
811 use super::*;
812 use crate::execution_store::ExecutionRecordOptions;
813 use serde_json::json;
814
815 fn executing_record() -> ExecutionRecord {
816 ExecutionRecord::with_options(ExecutionRecordOptions {
817 command: "sleep 60".to_string(),
818 uuid: Some("issue-126-rust".to_string()),
819 pid: Some(667105),
820 status: Some(ExecutionStatus::Executing),
821 log_path: Some("/tmp/issue-126.log".to_string()),
822 start_time: Some("2026-04-23T10:00:00Z".to_string()),
823 working_directory: Some("/home/user".to_string()),
824 shell: Some("/bin/bash".to_string()),
825 platform: Some("linux".to_string()),
826 ..Default::default()
827 })
828 }
829
830 #[test]
831 fn links_notation_indents_nested_process_id_arrays() {
832 let process_ids = json!({
833 "wrapperPid": 667105,
834 "screenPid": 667120,
835 "commandPids": [667121, 667122],
836 });
837 let output = format_record_with_enrichments(
838 &executing_record(),
839 "links-notation",
840 Some("2026-04-23T10:10:13.042Z"),
841 Some(&process_ids),
842 )
843 .expect("links-notation should format");
844
845 assert!(
846 output.contains(
847 " commandPids\n (\n 667121\n 667122\n )"
848 ),
849 "processIds should be a nested indented block, output: {}",
850 output
851 );
852 assert!(
853 !output.contains("\n(\n"),
854 "opening parenthesis must not start at column 1: {}",
855 output
856 );
857 }
858}