Skip to main content

codewhale_runtime/
session_diagnostics.rs

1//! Privacy-first session failure diagnostics (#2022).
2//!
3//! This module intentionally consumes loose JSONL event shapes instead of one
4//! exact persisted-session schema. Runtime logs, tool audits, and future bug
5//! exports can all emit slightly different records; the classifier only needs
6//! redacted handles, aggregate counts, and broad failure classes.
7
8use std::collections::{BTreeMap, BTreeSet};
9use std::fmt;
10
11use serde::{Deserialize, Serialize};
12use serde_json::Value;
13
14use codewhale_error_taxonomy::{ErrorCategory, classify_error_message};
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
17#[serde(rename_all = "snake_case")]
18pub enum SessionFailureClass {
19    CommandExit,
20    Network,
21    SandboxApproval,
22    MissingDependency,
23    Timeout,
24    BackgroundJob,
25    ToolSchema,
26    Model,
27    Unknown,
28    UnclosedTurn,
29}
30
31impl fmt::Display for SessionFailureClass {
32    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
33        let label = match self {
34            Self::CommandExit => "command_exit",
35            Self::Network => "network",
36            Self::SandboxApproval => "sandbox_approval",
37            Self::MissingDependency => "missing_dependency",
38            Self::Timeout => "timeout",
39            Self::BackgroundJob => "background_job",
40            Self::ToolSchema => "tool_schema",
41            Self::Model => "model",
42            Self::Unknown => "unknown",
43            Self::UnclosedTurn => "unclosed_turn",
44        };
45        f.write_str(label)
46    }
47}
48
49#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
50pub struct SessionFailureSource {
51    pub line: usize,
52    #[serde(skip_serializing_if = "Option::is_none")]
53    pub event: Option<String>,
54    #[serde(skip_serializing_if = "Option::is_none")]
55    pub turn_ref: Option<String>,
56    #[serde(skip_serializing_if = "Option::is_none")]
57    pub tool_name: Option<String>,
58    #[serde(skip_serializing_if = "Option::is_none")]
59    pub timestamp: Option<String>,
60}
61
62#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
63pub struct SessionFailureSummary {
64    pub total_lines: usize,
65    pub malformed_lines: usize,
66    pub counts: BTreeMap<SessionFailureClass, usize>,
67    pub sources: BTreeMap<SessionFailureClass, Vec<SessionFailureSource>>,
68}
69
70impl SessionFailureSummary {
71    #[must_use]
72    #[cfg_attr(not(test), expect(dead_code))]
73    pub(crate) fn count(&self, class: SessionFailureClass) -> usize {
74        self.counts.get(&class).copied().unwrap_or(0)
75    }
76
77    fn record(&mut self, class: SessionFailureClass, source: SessionFailureSource) {
78        *self.counts.entry(class).or_insert(0) += 1;
79        self.sources.entry(class).or_default().push(source);
80    }
81}
82
83#[must_use]
84pub fn analyze_session_failure_jsonl(jsonl: &str) -> SessionFailureSummary {
85    let mut summary = SessionFailureSummary {
86        total_lines: 0,
87        malformed_lines: 0,
88        counts: BTreeMap::new(),
89        sources: BTreeMap::new(),
90    };
91    let mut open_turns: BTreeMap<String, SessionFailureSource> = BTreeMap::new();
92
93    for (idx, raw_line) in jsonl.lines().enumerate() {
94        let line_no = idx + 1;
95        let trimmed = raw_line.trim();
96        if trimmed.is_empty() {
97            continue;
98        }
99        summary.total_lines += 1;
100        let Ok(value) = serde_json::from_str::<Value>(trimmed) else {
101            summary.malformed_lines += 1;
102            continue;
103        };
104
105        let event = event_name(&value);
106        let turn_id = string_field_any(&value, &["turn_id", "turnId", "run_id"]);
107        let source = source_handle(line_no, &value, event.clone(), turn_id.as_deref());
108        let failure_signal = has_failure_signal(&value);
109
110        if event_matches(
111            event.as_deref(),
112            &["turn_started", "turnstarted", "turn_start"],
113        ) {
114            if let Some(turn_id) = turn_id {
115                open_turns.insert(turn_id, source);
116            }
117            continue;
118        }
119        if event_matches(
120            event.as_deref(),
121            &[
122                "turn_complete",
123                "turncompleted",
124                "turn_finished",
125                "turnfinished",
126            ],
127        ) {
128            if let Some(turn_id) = turn_id.as_ref() {
129                open_turns.remove(turn_id);
130            }
131            if failure_signal {
132                let class = classify_failure_signal(&value);
133                summary.record(class, source);
134            }
135            continue;
136        }
137
138        if failure_signal {
139            let class = classify_failure_signal(&value);
140            summary.record(class, source);
141        }
142    }
143
144    for (_, source) in open_turns {
145        summary.record(SessionFailureClass::UnclosedTurn, source);
146    }
147
148    summary
149}
150
151#[must_use]
152pub fn format_redacted_failure_summary(summary: &SessionFailureSummary) -> String {
153    if summary.counts.is_empty() {
154        return "No session failure signals detected.".to_string();
155    }
156    let mut lines = vec![format!(
157        "Session failure diagnostics: {} JSONL lines inspected, {} malformed skipped.",
158        summary.total_lines, summary.malformed_lines
159    )];
160    for (class, count) in &summary.counts {
161        let sample = summary
162            .sources
163            .get(class)
164            .and_then(|sources| sources.first())
165            .map(format_source)
166            .unwrap_or_else(|| "no source".to_string());
167        lines.push(format!("- {class}: {count} (sample: {sample})"));
168    }
169    lines.join("\n")
170}
171
172fn source_handle(
173    line: usize,
174    value: &Value,
175    event: Option<String>,
176    turn_id: Option<&str>,
177) -> SessionFailureSource {
178    let tool_name = string_field_any(value, &["tool_name", "toolName", "tool"])
179        .filter(|name| event.as_deref().is_none_or(|event| event != name));
180    SessionFailureSource {
181        line,
182        event,
183        turn_ref: turn_id.map(codewhale_util::utils::redacted_identifier_for_log),
184        tool_name,
185        timestamp: string_field_any(value, &["timestamp", "ts", "created_at", "createdAt"]),
186    }
187}
188
189fn format_source(source: &SessionFailureSource) -> String {
190    let mut parts = vec![format!("line {}", source.line)];
191    if let Some(event) = source.event.as_deref() {
192        parts.push(format!("event={event}"));
193    }
194    if let Some(turn_ref) = source.turn_ref.as_deref() {
195        parts.push(format!("turn={turn_ref}"));
196    }
197    if let Some(tool_name) = source.tool_name.as_deref() {
198        parts.push(format!("tool={tool_name}"));
199    }
200    if let Some(timestamp) = source.timestamp.as_deref() {
201        parts.push(format!("ts={timestamp}"));
202    }
203    parts.join(" ")
204}
205
206fn has_failure_signal(value: &Value) -> bool {
207    numeric_field_any(value, &["exit_code", "exitCode"]).is_some_and(|code| code != 0)
208        || bool_field_any(value, &["success"]).is_some_and(|success| !success)
209        || bool_field_any(value, &["is_error", "isError"]).unwrap_or(false)
210        || failure_status(value).is_some()
211        || string_field_any(value, &["error", "stderr"]).is_some_and(|text| !text.is_empty())
212}
213
214fn classify_failure_signal(value: &Value) -> SessionFailureClass {
215    if let Some(message) = diagnostic_message(value) {
216        return classify_session_failure(value, &message);
217    }
218    if let Some(status) = failure_status(value) {
219        let lower = status.to_ascii_lowercase();
220        if lower.contains("timeout") || lower.contains("timed_out") {
221            return SessionFailureClass::Timeout;
222        }
223        if lower.contains("cancel") || lower.contains("background") || lower.contains("stale") {
224            return SessionFailureClass::BackgroundJob;
225        }
226    }
227    if numeric_field_any(value, &["exit_code", "exitCode"]).is_some_and(|code| code != 0) {
228        return SessionFailureClass::CommandExit;
229    }
230    SessionFailureClass::Unknown
231}
232
233fn classify_session_failure(value: &Value, message: &str) -> SessionFailureClass {
234    let lower = message.to_ascii_lowercase();
235    if lower.contains("background")
236        || lower.contains("task_shell")
237        || lower.contains("job timed out")
238        || lower.contains("job cancelled")
239        || lower.contains("stale job")
240    {
241        return SessionFailureClass::BackgroundJob;
242    }
243    if lower.contains("sandbox")
244        || lower.contains("approval")
245        || lower.contains("permission denied")
246        || lower.contains("operation not permitted")
247        || lower.contains("read-only")
248        || lower.contains("access is denied")
249    {
250        return SessionFailureClass::SandboxApproval;
251    }
252    if lower.contains("command not found")
253        || lower.contains("no such file or directory")
254        || lower.contains("missing binary")
255        || lower.contains("enoent")
256        || lower.contains("not installed")
257    {
258        return SessionFailureClass::MissingDependency;
259    }
260    if lower.contains("missing field")
261        || lower.contains("invalid tool")
262        || lower.contains("invalid input")
263        || lower.contains("schema")
264        || lower.contains("tool arguments")
265    {
266        return SessionFailureClass::ToolSchema;
267    }
268    if numeric_field_any(value, &["exit_code", "exitCode"]).is_some_and(|code| code != 0)
269        || lower.contains("non-zero")
270        || lower.contains("exit status")
271        || lower.contains("exit code")
272    {
273        return SessionFailureClass::CommandExit;
274    }
275    match classify_error_message(message) {
276        ErrorCategory::Network | ErrorCategory::RateLimit => SessionFailureClass::Network,
277        ErrorCategory::Timeout => SessionFailureClass::Timeout,
278        ErrorCategory::Budget => SessionFailureClass::Unknown,
279        ErrorCategory::Authorization => SessionFailureClass::SandboxApproval,
280        ErrorCategory::Authentication => SessionFailureClass::Model,
281        ErrorCategory::State => SessionFailureClass::MissingDependency,
282        // A bare placeholder ("error") says nothing about a schema fault; the
283        // classifier files it under Parse only so provider turns can say
284        // "unreadable" (#6843).
285        ErrorCategory::Parse
286            if codewhale_error_taxonomy::unreadable_error_notice(message).is_some() =>
287        {
288            SessionFailureClass::Unknown
289        }
290        ErrorCategory::InvalidInput | ErrorCategory::Parse => SessionFailureClass::ToolSchema,
291        ErrorCategory::Tool => SessionFailureClass::CommandExit,
292        ErrorCategory::Internal if lower.contains("model") => SessionFailureClass::Model,
293        ErrorCategory::Internal => SessionFailureClass::Unknown,
294    }
295}
296
297fn diagnostic_message(value: &Value) -> Option<String> {
298    let mut parts = Vec::new();
299    collect_string_fields(
300        value,
301        &mut parts,
302        &[
303            "error", "message", "stderr", "reason", "result", "output", "content",
304        ],
305        0,
306    );
307    let mut seen = BTreeSet::new();
308    let deduped = parts
309        .into_iter()
310        .filter(|part| !part.trim().is_empty())
311        .filter(|part| seen.insert(part.clone()))
312        .collect::<Vec<_>>();
313    (!deduped.is_empty()).then(|| deduped.join(" "))
314}
315
316fn collect_string_fields(value: &Value, out: &mut Vec<String>, keys: &[&str], depth: usize) {
317    if depth > 4 {
318        return;
319    }
320    match value {
321        Value::Object(map) => {
322            for (key, value) in map {
323                if keys
324                    .iter()
325                    .any(|candidate| key.eq_ignore_ascii_case(candidate))
326                    && let Some(text) = value.as_str()
327                {
328                    out.push(text.to_string());
329                }
330                collect_string_fields(value, out, keys, depth + 1);
331            }
332        }
333        Value::Array(items) => {
334            for item in items {
335                collect_string_fields(item, out, keys, depth + 1);
336            }
337        }
338        _ => {}
339    }
340}
341
342fn event_name(value: &Value) -> Option<String> {
343    string_field_any(value, &["event", "type", "kind"]).map(|event| normalize_event(&event))
344}
345
346fn normalize_event(event: &str) -> String {
347    event
348        .trim()
349        .trim_matches('"')
350        .replace(['-', ' ', '.'], "_")
351        .to_ascii_lowercase()
352}
353
354fn event_matches(event: Option<&str>, aliases: &[&str]) -> bool {
355    event.is_some_and(|event| aliases.contains(&event))
356}
357
358fn failure_status(value: &Value) -> Option<String> {
359    string_field_any(value, &["status", "state", "outcome"]).filter(|status| {
360        let normalized = normalize_event(status);
361        matches!(
362            normalized.as_str(),
363            "failed"
364                | "failure"
365                | "error"
366                | "errored"
367                | "cancelled"
368                | "canceled"
369                | "timeout"
370                | "timed_out"
371                | "stale"
372        )
373    })
374}
375
376fn string_field_any(value: &Value, keys: &[&str]) -> Option<String> {
377    string_field_any_at(value, keys, 0)
378}
379
380fn string_field_any_at(value: &Value, keys: &[&str], depth: usize) -> Option<String> {
381    if depth > 4 {
382        return None;
383    }
384    match value {
385        Value::Object(map) => {
386            for key in keys {
387                if let Some(value) = map.iter().find_map(|(candidate, value)| {
388                    candidate.eq_ignore_ascii_case(key).then_some(value)
389                }) && let Some(text) = value.as_str()
390                {
391                    return Some(text.to_string());
392                }
393            }
394            for child in map.values() {
395                if let Some(found) = string_field_any_at(child, keys, depth + 1) {
396                    return Some(found);
397                }
398            }
399            None
400        }
401        Value::Array(items) => items
402            .iter()
403            .find_map(|item| string_field_any_at(item, keys, depth + 1)),
404        _ => None,
405    }
406}
407
408fn numeric_field_any(value: &Value, keys: &[&str]) -> Option<i64> {
409    numeric_field_any_at(value, keys, 0)
410}
411
412fn numeric_field_any_at(value: &Value, keys: &[&str], depth: usize) -> Option<i64> {
413    if depth > 4 {
414        return None;
415    }
416    match value {
417        Value::Object(map) => {
418            for key in keys {
419                if let Some(value) = map.iter().find_map(|(candidate, value)| {
420                    candidate.eq_ignore_ascii_case(key).then_some(value)
421                }) && let Some(number) = value.as_i64()
422                {
423                    return Some(number);
424                }
425            }
426            for child in map.values() {
427                if let Some(found) = numeric_field_any_at(child, keys, depth + 1) {
428                    return Some(found);
429                }
430            }
431            None
432        }
433        Value::Array(items) => items
434            .iter()
435            .find_map(|item| numeric_field_any_at(item, keys, depth + 1)),
436        _ => None,
437    }
438}
439
440fn bool_field_any(value: &Value, keys: &[&str]) -> Option<bool> {
441    bool_field_any_at(value, keys, 0)
442}
443
444fn bool_field_any_at(value: &Value, keys: &[&str], depth: usize) -> Option<bool> {
445    if depth > 4 {
446        return None;
447    }
448    match value {
449        Value::Object(map) => {
450            for key in keys {
451                if let Some(value) = map.iter().find_map(|(candidate, value)| {
452                    candidate.eq_ignore_ascii_case(key).then_some(value)
453                }) && let Some(flag) = value.as_bool()
454                {
455                    return Some(flag);
456                }
457            }
458            for child in map.values() {
459                if let Some(found) = bool_field_any_at(child, keys, depth + 1) {
460                    return Some(found);
461                }
462            }
463            None
464        }
465        Value::Array(items) => items
466            .iter()
467            .find_map(|item| bool_field_any_at(item, keys, depth + 1)),
468        _ => None,
469    }
470}
471
472#[cfg(test)]
473mod tests {
474    use super::*;
475
476    #[test]
477    fn bare_error_placeholder_is_not_a_schema_failure() {
478        // The taxonomy files a bare "error" under Parse so provider turns can
479        // call it unreadable; a tool that printed only that is not a schema bug.
480        let value = serde_json::json!({});
481        assert_eq!(
482            classify_session_failure(&value, "ERROR"),
483            SessionFailureClass::Unknown
484        );
485    }
486
487    #[test]
488    fn synthetic_jsonl_classifies_environment_and_tool_failures() {
489        let jsonl = r#"
490{"event":"turn_started","turn_id":"turn-secret-1","timestamp":"2026-06-25T12:00:00Z"}
491{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"exec_shell","exit_code":127,"stderr":"bash: rg: command not found"}
492{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"exec_shell","exit_code":2,"stderr":"command failed with exit code 2"}
493{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"web_search","error":"DNS resolution failed for api.example.test"}
494{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"apply_patch","error":"Permission denied by sandbox: read-only filesystem"}
495{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"exec_shell","error":"request timed out after 30s"}
496{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"task_shell_wait","error":"background job timed out"}
497{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"mcp_tool","error":"missing field: tool arguments"}
498{"event":"turn_complete","turn_id":"turn-secret-1","status":"completed"}
499{"event":"turn_started","turn_id":"turn-secret-2"}
500not json at all
501"#;
502
503        let summary = analyze_session_failure_jsonl(jsonl);
504
505        assert_eq!(summary.malformed_lines, 1);
506        assert_eq!(summary.count(SessionFailureClass::MissingDependency), 1);
507        assert_eq!(summary.count(SessionFailureClass::CommandExit), 1);
508        assert_eq!(summary.count(SessionFailureClass::Network), 1);
509        assert_eq!(summary.count(SessionFailureClass::SandboxApproval), 1);
510        assert_eq!(summary.count(SessionFailureClass::Timeout), 1);
511        assert_eq!(summary.count(SessionFailureClass::BackgroundJob), 1);
512        assert_eq!(summary.count(SessionFailureClass::ToolSchema), 1);
513        assert_eq!(summary.count(SessionFailureClass::UnclosedTurn), 1);
514
515        let sources = summary
516            .sources
517            .get(&SessionFailureClass::MissingDependency)
518            .expect("missing-dependency source");
519        assert_eq!(sources[0].tool_name.as_deref(), Some("exec_shell"));
520        assert!(
521            sources[0]
522                .turn_ref
523                .as_deref()
524                .is_some_and(|turn| turn.starts_with("<redacted:")),
525            "turn ids must be redacted: {sources:?}"
526        );
527    }
528
529    #[test]
530    fn redacted_summary_omits_raw_messages_and_paths() {
531        let jsonl = r#"
532{"event":"turn_started","turn_id":"turn-secret-1"}
533{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"read_file","timestamp":"2026-06-25T12:34:56Z","error":"No such file or directory: /Users/alice/secret/project/.env"}
534"#;
535
536        let summary = analyze_session_failure_jsonl(jsonl);
537        let rendered = format_redacted_failure_summary(&summary);
538
539        assert!(rendered.contains("missing_dependency"));
540        assert!(rendered.contains("line 3"));
541        assert!(rendered.contains("tool=read_file"));
542        assert!(rendered.contains("ts=2026-06-25T12:34:56Z"));
543        assert!(!rendered.contains("alice"));
544        assert!(!rendered.contains(".env"));
545        assert!(!rendered.contains("turn-secret-1"));
546    }
547
548    #[test]
549    fn successful_content_does_not_create_unknown_failure() {
550        let jsonl = r#"
551{"event":"turn_started","turn_id":"turn-secret-1"}
552{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"exec_shell","success":true,"content":"command output mentioning error budgets is still normal content"}
553{"event":"turn_complete","turn_id":"turn-secret-1","status":"completed","message":"done"}
554"#;
555
556        let summary = analyze_session_failure_jsonl(jsonl);
557
558        assert_eq!(summary.count(SessionFailureClass::Unknown), 0);
559        assert_eq!(summary.count(SessionFailureClass::CommandExit), 0);
560        assert_eq!(summary.count(SessionFailureClass::UnclosedTurn), 0);
561        assert!(
562            summary.counts.is_empty(),
563            "summary should be empty: {summary:?}"
564        );
565    }
566
567    #[test]
568    fn empty_error_field_is_not_a_failure_signal() {
569        let jsonl = r#"
570{"event":"tool_call_complete","success":true,"error":"","stderr":""}
571"#;
572
573        let summary = analyze_session_failure_jsonl(jsonl);
574
575        assert!(
576            summary.counts.is_empty(),
577            "empty error/stderr sentinels should not signal failure: {summary:?}"
578        );
579    }
580
581    #[test]
582    fn nested_generic_fields_do_not_shadow_source_handles() {
583        let jsonl = r#"
584{"event":"tool_call_complete","turn_id":"turn-real","tool_name":"exec_shell","success":false,"payload":{"id":"nested-id","name":"nested-name","error":"nested error"}}
585"#;
586
587        let summary = analyze_session_failure_jsonl(jsonl);
588        let source = summary
589            .sources
590            .values()
591            .flat_map(|sources| sources.iter())
592            .next()
593            .expect("failure source");
594
595        assert_eq!(source.tool_name.as_deref(), Some("exec_shell"));
596        assert!(
597            source
598                .turn_ref
599                .as_deref()
600                .is_some_and(|turn| turn.starts_with("<redacted:")),
601            "top-level turn id should be redacted and used: {source:?}"
602        );
603    }
604
605    #[test]
606    fn deeply_nested_error_field_is_ignored() {
607        let jsonl = r#"
608{"event":"tool_call_complete","payload":{"a":{"b":{"c":{"d":{"e":{"error":"too deep"}}}}}}}
609"#;
610
611        let summary = analyze_session_failure_jsonl(jsonl);
612
613        assert!(
614            summary.counts.is_empty(),
615            "deeply nested error fields should not signal failure: {summary:?}"
616        );
617    }
618
619    #[test]
620    fn authentication_failures_are_model_failures_not_sandbox_approval() {
621        let jsonl = r#"
622{"event":"model_response","success":false,"error":"Authentication failed: invalid API key"}
623"#;
624
625        let summary = analyze_session_failure_jsonl(jsonl);
626
627        assert_eq!(summary.count(SessionFailureClass::Model), 1);
628        assert_eq!(summary.count(SessionFailureClass::SandboxApproval), 0);
629    }
630}