Skip to main content

codewhale_runtime/
stall_report.rs

1use std::path::PathBuf;
2use std::time::Duration;
3
4/// One detected stall episode.
5#[derive(Debug, Clone, PartialEq, Eq)]
6pub struct StallReport {
7    /// Which watchdog saw it (`engine`, `ui`, `client`).
8    pub source: &'static str,
9    pub phase: String,
10    pub detail: Option<String>,
11    pub turn_id: Option<String>,
12    /// Provider response/request id when the stream reported one, else the
13    /// route label the request went to.
14    pub provider_request: Option<String>,
15    pub since_progress: Duration,
16    pub bound: Option<Duration>,
17}
18
19impl StallReport {
20    /// One user-facing line: where it stalled and what to do.
21    #[must_use]
22    pub fn status_line(&self) -> String {
23        let mut line = format!(
24            "Turn stalled {} — no progress for {}s",
25            self.phase,
26            self.since_progress.as_secs()
27        );
28        if let Some(detail) = self.detail.as_deref().filter(|d| !d.is_empty()) {
29            line.push_str(&format!(" ({detail})"));
30        }
31        line.push_str(". Press Esc to cancel and retry.");
32        line
33    }
34
35    fn record_body(&self) -> String {
36        let timestamp = chrono::Utc::now().to_rfc3339();
37        let bound = self.bound.map_or_else(
38            || "none (parked)".to_string(),
39            |b| format!("{}s", b.as_secs()),
40        );
41        format!(
42            "Kind: turn-stall\nSource: {source}\nTimestamp: {timestamp}\nPhase: {phase}\n\
43             Detail: {detail}\nTurn: {turn}\nProvider request: {request}\n\
44             No progress for: {since}s\nPhase bound: {bound}\n",
45            source = self.source,
46            phase = self.phase,
47            detail = self.detail.as_deref().unwrap_or("-"),
48            turn = self.turn_id.as_deref().unwrap_or("-"),
49            request = self.provider_request.as_deref().unwrap_or("-"),
50            since = self.since_progress.as_secs(),
51        )
52    }
53}
54
55#[cfg(any(test, all(feature = "test-thread-scoped-state", debug_assertions)))]
56thread_local! {
57    static TEST_STALL_RECORD_DIR: std::cell::RefCell<Option<PathBuf>> =
58        const { std::cell::RefCell::new(None) };
59}
60
61/// Route stall records for the current test thread into `dir` (tests never
62/// write into the real `~/.codewhale/crashes`).
63#[cfg(any(test, all(feature = "test-thread-scoped-state", debug_assertions)))]
64pub fn set_test_stall_record_dir(dir: Option<PathBuf>) {
65    TEST_STALL_RECORD_DIR.with(|slot| *slot.borrow_mut() = dir);
66}
67
68/// The selected profile's crash directory, shared with panic dumps.
69pub fn stall_record_dir() -> Option<PathBuf> {
70    #[cfg(any(test, all(feature = "test-thread-scoped-state", debug_assertions)))]
71    {
72        TEST_STALL_RECORD_DIR.with(|slot| slot.borrow().clone())
73    }
74    #[cfg(not(any(test, all(feature = "test-thread-scoped-state", debug_assertions))))]
75    {
76        codewhale_config::codewhale_home()
77            .ok()
78            .map(|home| home.join("crashes"))
79    }
80}
81
82/// Log a stall and write its record to `crashes/<timestamp>-turn-stall-<source>.log`.
83/// Best effort; returns the record path when a record directory exists. The
84/// write runs on its own short-lived thread so no caller (engine task or UI
85/// event loop) blocks a runtime worker on disk I/O (#6149).
86pub fn report_stall(report: &StallReport) -> Option<PathBuf> {
87    let path = stall_record_dir().map(|dir| {
88        let stamp = chrono::Utc::now().format("%Y%m%dT%H%M%S%.3fZ");
89        dir.join(format!("{stamp}-turn-stall-{}.log", report.source))
90    });
91    if let Some(path) = path.clone() {
92        let body = report.record_body();
93        let writer = std::thread::spawn(move || {
94            if let Some(dir) = path.parent() {
95                let _ = std::fs::create_dir_all(dir);
96            }
97            let _ = std::fs::write(&path, body);
98        });
99        // Tests read the record right after reporting.
100        #[cfg(any(test, all(feature = "test-thread-scoped-state", debug_assertions)))]
101        let _ = writer.join();
102        #[cfg(not(any(test, all(feature = "test-thread-scoped-state", debug_assertions))))]
103        drop(writer);
104    }
105    let message = format!(
106        "turn stall ({source}): phase={phase} since_progress={since}s bound={bound:?} turn={turn} request={request} detail={detail} record={record}",
107        source = report.source,
108        phase = report.phase,
109        since = report.since_progress.as_secs(),
110        bound = report.bound.map(|b| b.as_secs()),
111        turn = report.turn_id.as_deref().unwrap_or("-"),
112        request = report.provider_request.as_deref().unwrap_or("-"),
113        detail = report.detail.as_deref().unwrap_or("-"),
114        record = path
115            .as_deref()
116            .map_or_else(|| "-".to_string(), |p| p.display().to_string()),
117    );
118    tracing::warn!(target: "turn_stall", "{message}");
119    crate::logging::warn(&message);
120    path
121}