1use std::collections::{BTreeMap, BTreeSet};
9use std::fmt;
10
11use serde::{Deserialize, Serialize};
12use serde_json::Value;
13
14use codewhale_error_taxonomy::{ErrorCategory, classify_error_message};
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
17#[serde(rename_all = "snake_case")]
18pub enum SessionFailureClass {
19 CommandExit,
20 Network,
21 SandboxApproval,
22 MissingDependency,
23 Timeout,
24 BackgroundJob,
25 ToolSchema,
26 Model,
27 Unknown,
28 UnclosedTurn,
29}
30
31impl fmt::Display for SessionFailureClass {
32 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
33 let label = match self {
34 Self::CommandExit => "command_exit",
35 Self::Network => "network",
36 Self::SandboxApproval => "sandbox_approval",
37 Self::MissingDependency => "missing_dependency",
38 Self::Timeout => "timeout",
39 Self::BackgroundJob => "background_job",
40 Self::ToolSchema => "tool_schema",
41 Self::Model => "model",
42 Self::Unknown => "unknown",
43 Self::UnclosedTurn => "unclosed_turn",
44 };
45 f.write_str(label)
46 }
47}
48
49#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
50pub struct SessionFailureSource {
51 pub line: usize,
52 #[serde(skip_serializing_if = "Option::is_none")]
53 pub event: Option<String>,
54 #[serde(skip_serializing_if = "Option::is_none")]
55 pub turn_ref: Option<String>,
56 #[serde(skip_serializing_if = "Option::is_none")]
57 pub tool_name: Option<String>,
58 #[serde(skip_serializing_if = "Option::is_none")]
59 pub timestamp: Option<String>,
60}
61
62#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
63pub struct SessionFailureSummary {
64 pub total_lines: usize,
65 pub malformed_lines: usize,
66 pub counts: BTreeMap<SessionFailureClass, usize>,
67 pub sources: BTreeMap<SessionFailureClass, Vec<SessionFailureSource>>,
68}
69
70impl SessionFailureSummary {
71 #[must_use]
72 #[cfg_attr(not(test), expect(dead_code))]
73 pub(crate) fn count(&self, class: SessionFailureClass) -> usize {
74 self.counts.get(&class).copied().unwrap_or(0)
75 }
76
77 fn record(&mut self, class: SessionFailureClass, source: SessionFailureSource) {
78 *self.counts.entry(class).or_insert(0) += 1;
79 self.sources.entry(class).or_default().push(source);
80 }
81}
82
83#[must_use]
84pub fn analyze_session_failure_jsonl(jsonl: &str) -> SessionFailureSummary {
85 let mut summary = SessionFailureSummary {
86 total_lines: 0,
87 malformed_lines: 0,
88 counts: BTreeMap::new(),
89 sources: BTreeMap::new(),
90 };
91 let mut open_turns: BTreeMap<String, SessionFailureSource> = BTreeMap::new();
92
93 for (idx, raw_line) in jsonl.lines().enumerate() {
94 let line_no = idx + 1;
95 let trimmed = raw_line.trim();
96 if trimmed.is_empty() {
97 continue;
98 }
99 summary.total_lines += 1;
100 let Ok(value) = serde_json::from_str::<Value>(trimmed) else {
101 summary.malformed_lines += 1;
102 continue;
103 };
104
105 let event = event_name(&value);
106 let turn_id = string_field_any(&value, &["turn_id", "turnId", "run_id"]);
107 let source = source_handle(line_no, &value, event.clone(), turn_id.as_deref());
108 let failure_signal = has_failure_signal(&value);
109
110 if event_matches(
111 event.as_deref(),
112 &["turn_started", "turnstarted", "turn_start"],
113 ) {
114 if let Some(turn_id) = turn_id {
115 open_turns.insert(turn_id, source);
116 }
117 continue;
118 }
119 if event_matches(
120 event.as_deref(),
121 &[
122 "turn_complete",
123 "turncompleted",
124 "turn_finished",
125 "turnfinished",
126 ],
127 ) {
128 if let Some(turn_id) = turn_id.as_ref() {
129 open_turns.remove(turn_id);
130 }
131 if failure_signal {
132 let class = classify_failure_signal(&value);
133 summary.record(class, source);
134 }
135 continue;
136 }
137
138 if failure_signal {
139 let class = classify_failure_signal(&value);
140 summary.record(class, source);
141 }
142 }
143
144 for (_, source) in open_turns {
145 summary.record(SessionFailureClass::UnclosedTurn, source);
146 }
147
148 summary
149}
150
151#[must_use]
152pub fn format_redacted_failure_summary(summary: &SessionFailureSummary) -> String {
153 if summary.counts.is_empty() {
154 return "No session failure signals detected.".to_string();
155 }
156 let mut lines = vec![format!(
157 "Session failure diagnostics: {} JSONL lines inspected, {} malformed skipped.",
158 summary.total_lines, summary.malformed_lines
159 )];
160 for (class, count) in &summary.counts {
161 let sample = summary
162 .sources
163 .get(class)
164 .and_then(|sources| sources.first())
165 .map(format_source)
166 .unwrap_or_else(|| "no source".to_string());
167 lines.push(format!("- {class}: {count} (sample: {sample})"));
168 }
169 lines.join("\n")
170}
171
172fn source_handle(
173 line: usize,
174 value: &Value,
175 event: Option<String>,
176 turn_id: Option<&str>,
177) -> SessionFailureSource {
178 let tool_name = string_field_any(value, &["tool_name", "toolName", "tool"])
179 .filter(|name| event.as_deref().is_none_or(|event| event != name));
180 SessionFailureSource {
181 line,
182 event,
183 turn_ref: turn_id.map(codewhale_util::utils::redacted_identifier_for_log),
184 tool_name,
185 timestamp: string_field_any(value, &["timestamp", "ts", "created_at", "createdAt"]),
186 }
187}
188
189fn format_source(source: &SessionFailureSource) -> String {
190 let mut parts = vec![format!("line {}", source.line)];
191 if let Some(event) = source.event.as_deref() {
192 parts.push(format!("event={event}"));
193 }
194 if let Some(turn_ref) = source.turn_ref.as_deref() {
195 parts.push(format!("turn={turn_ref}"));
196 }
197 if let Some(tool_name) = source.tool_name.as_deref() {
198 parts.push(format!("tool={tool_name}"));
199 }
200 if let Some(timestamp) = source.timestamp.as_deref() {
201 parts.push(format!("ts={timestamp}"));
202 }
203 parts.join(" ")
204}
205
206fn has_failure_signal(value: &Value) -> bool {
207 numeric_field_any(value, &["exit_code", "exitCode"]).is_some_and(|code| code != 0)
208 || bool_field_any(value, &["success"]).is_some_and(|success| !success)
209 || bool_field_any(value, &["is_error", "isError"]).unwrap_or(false)
210 || failure_status(value).is_some()
211 || string_field_any(value, &["error", "stderr"]).is_some_and(|text| !text.is_empty())
212}
213
214fn classify_failure_signal(value: &Value) -> SessionFailureClass {
215 if let Some(message) = diagnostic_message(value) {
216 return classify_session_failure(value, &message);
217 }
218 if let Some(status) = failure_status(value) {
219 let lower = status.to_ascii_lowercase();
220 if lower.contains("timeout") || lower.contains("timed_out") {
221 return SessionFailureClass::Timeout;
222 }
223 if lower.contains("cancel") || lower.contains("background") || lower.contains("stale") {
224 return SessionFailureClass::BackgroundJob;
225 }
226 }
227 if numeric_field_any(value, &["exit_code", "exitCode"]).is_some_and(|code| code != 0) {
228 return SessionFailureClass::CommandExit;
229 }
230 SessionFailureClass::Unknown
231}
232
233fn classify_session_failure(value: &Value, message: &str) -> SessionFailureClass {
234 let lower = message.to_ascii_lowercase();
235 if lower.contains("background")
236 || lower.contains("task_shell")
237 || lower.contains("job timed out")
238 || lower.contains("job cancelled")
239 || lower.contains("stale job")
240 {
241 return SessionFailureClass::BackgroundJob;
242 }
243 if lower.contains("sandbox")
244 || lower.contains("approval")
245 || lower.contains("permission denied")
246 || lower.contains("operation not permitted")
247 || lower.contains("read-only")
248 || lower.contains("access is denied")
249 {
250 return SessionFailureClass::SandboxApproval;
251 }
252 if lower.contains("command not found")
253 || lower.contains("no such file or directory")
254 || lower.contains("missing binary")
255 || lower.contains("enoent")
256 || lower.contains("not installed")
257 {
258 return SessionFailureClass::MissingDependency;
259 }
260 if lower.contains("missing field")
261 || lower.contains("invalid tool")
262 || lower.contains("invalid input")
263 || lower.contains("schema")
264 || lower.contains("tool arguments")
265 {
266 return SessionFailureClass::ToolSchema;
267 }
268 if numeric_field_any(value, &["exit_code", "exitCode"]).is_some_and(|code| code != 0)
269 || lower.contains("non-zero")
270 || lower.contains("exit status")
271 || lower.contains("exit code")
272 {
273 return SessionFailureClass::CommandExit;
274 }
275 match classify_error_message(message) {
276 ErrorCategory::Network | ErrorCategory::RateLimit => SessionFailureClass::Network,
277 ErrorCategory::Timeout => SessionFailureClass::Timeout,
278 ErrorCategory::Budget => SessionFailureClass::Unknown,
279 ErrorCategory::Authorization => SessionFailureClass::SandboxApproval,
280 ErrorCategory::Authentication => SessionFailureClass::Model,
281 ErrorCategory::State => SessionFailureClass::MissingDependency,
282 ErrorCategory::Parse
286 if codewhale_error_taxonomy::unreadable_error_notice(message).is_some() =>
287 {
288 SessionFailureClass::Unknown
289 }
290 ErrorCategory::InvalidInput | ErrorCategory::Parse => SessionFailureClass::ToolSchema,
291 ErrorCategory::Tool => SessionFailureClass::CommandExit,
292 ErrorCategory::Internal if lower.contains("model") => SessionFailureClass::Model,
293 ErrorCategory::Internal => SessionFailureClass::Unknown,
294 }
295}
296
297fn diagnostic_message(value: &Value) -> Option<String> {
298 let mut parts = Vec::new();
299 collect_string_fields(
300 value,
301 &mut parts,
302 &[
303 "error", "message", "stderr", "reason", "result", "output", "content",
304 ],
305 0,
306 );
307 let mut seen = BTreeSet::new();
308 let deduped = parts
309 .into_iter()
310 .filter(|part| !part.trim().is_empty())
311 .filter(|part| seen.insert(part.clone()))
312 .collect::<Vec<_>>();
313 (!deduped.is_empty()).then(|| deduped.join(" "))
314}
315
316fn collect_string_fields(value: &Value, out: &mut Vec<String>, keys: &[&str], depth: usize) {
317 if depth > 4 {
318 return;
319 }
320 match value {
321 Value::Object(map) => {
322 for (key, value) in map {
323 if keys
324 .iter()
325 .any(|candidate| key.eq_ignore_ascii_case(candidate))
326 && let Some(text) = value.as_str()
327 {
328 out.push(text.to_string());
329 }
330 collect_string_fields(value, out, keys, depth + 1);
331 }
332 }
333 Value::Array(items) => {
334 for item in items {
335 collect_string_fields(item, out, keys, depth + 1);
336 }
337 }
338 _ => {}
339 }
340}
341
342fn event_name(value: &Value) -> Option<String> {
343 string_field_any(value, &["event", "type", "kind"]).map(|event| normalize_event(&event))
344}
345
346fn normalize_event(event: &str) -> String {
347 event
348 .trim()
349 .trim_matches('"')
350 .replace(['-', ' ', '.'], "_")
351 .to_ascii_lowercase()
352}
353
354fn event_matches(event: Option<&str>, aliases: &[&str]) -> bool {
355 event.is_some_and(|event| aliases.contains(&event))
356}
357
358fn failure_status(value: &Value) -> Option<String> {
359 string_field_any(value, &["status", "state", "outcome"]).filter(|status| {
360 let normalized = normalize_event(status);
361 matches!(
362 normalized.as_str(),
363 "failed"
364 | "failure"
365 | "error"
366 | "errored"
367 | "cancelled"
368 | "canceled"
369 | "timeout"
370 | "timed_out"
371 | "stale"
372 )
373 })
374}
375
376fn string_field_any(value: &Value, keys: &[&str]) -> Option<String> {
377 string_field_any_at(value, keys, 0)
378}
379
380fn string_field_any_at(value: &Value, keys: &[&str], depth: usize) -> Option<String> {
381 if depth > 4 {
382 return None;
383 }
384 match value {
385 Value::Object(map) => {
386 for key in keys {
387 if let Some(value) = map.iter().find_map(|(candidate, value)| {
388 candidate.eq_ignore_ascii_case(key).then_some(value)
389 }) && let Some(text) = value.as_str()
390 {
391 return Some(text.to_string());
392 }
393 }
394 for child in map.values() {
395 if let Some(found) = string_field_any_at(child, keys, depth + 1) {
396 return Some(found);
397 }
398 }
399 None
400 }
401 Value::Array(items) => items
402 .iter()
403 .find_map(|item| string_field_any_at(item, keys, depth + 1)),
404 _ => None,
405 }
406}
407
408fn numeric_field_any(value: &Value, keys: &[&str]) -> Option<i64> {
409 numeric_field_any_at(value, keys, 0)
410}
411
412fn numeric_field_any_at(value: &Value, keys: &[&str], depth: usize) -> Option<i64> {
413 if depth > 4 {
414 return None;
415 }
416 match value {
417 Value::Object(map) => {
418 for key in keys {
419 if let Some(value) = map.iter().find_map(|(candidate, value)| {
420 candidate.eq_ignore_ascii_case(key).then_some(value)
421 }) && let Some(number) = value.as_i64()
422 {
423 return Some(number);
424 }
425 }
426 for child in map.values() {
427 if let Some(found) = numeric_field_any_at(child, keys, depth + 1) {
428 return Some(found);
429 }
430 }
431 None
432 }
433 Value::Array(items) => items
434 .iter()
435 .find_map(|item| numeric_field_any_at(item, keys, depth + 1)),
436 _ => None,
437 }
438}
439
440fn bool_field_any(value: &Value, keys: &[&str]) -> Option<bool> {
441 bool_field_any_at(value, keys, 0)
442}
443
444fn bool_field_any_at(value: &Value, keys: &[&str], depth: usize) -> Option<bool> {
445 if depth > 4 {
446 return None;
447 }
448 match value {
449 Value::Object(map) => {
450 for key in keys {
451 if let Some(value) = map.iter().find_map(|(candidate, value)| {
452 candidate.eq_ignore_ascii_case(key).then_some(value)
453 }) && let Some(flag) = value.as_bool()
454 {
455 return Some(flag);
456 }
457 }
458 for child in map.values() {
459 if let Some(found) = bool_field_any_at(child, keys, depth + 1) {
460 return Some(found);
461 }
462 }
463 None
464 }
465 Value::Array(items) => items
466 .iter()
467 .find_map(|item| bool_field_any_at(item, keys, depth + 1)),
468 _ => None,
469 }
470}
471
472#[cfg(test)]
473mod tests {
474 use super::*;
475
476 #[test]
477 fn bare_error_placeholder_is_not_a_schema_failure() {
478 let value = serde_json::json!({});
481 assert_eq!(
482 classify_session_failure(&value, "ERROR"),
483 SessionFailureClass::Unknown
484 );
485 }
486
487 #[test]
488 fn synthetic_jsonl_classifies_environment_and_tool_failures() {
489 let jsonl = r#"
490{"event":"turn_started","turn_id":"turn-secret-1","timestamp":"2026-06-25T12:00:00Z"}
491{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"exec_shell","exit_code":127,"stderr":"bash: rg: command not found"}
492{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"exec_shell","exit_code":2,"stderr":"command failed with exit code 2"}
493{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"web_search","error":"DNS resolution failed for api.example.test"}
494{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"apply_patch","error":"Permission denied by sandbox: read-only filesystem"}
495{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"exec_shell","error":"request timed out after 30s"}
496{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"task_shell_wait","error":"background job timed out"}
497{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"mcp_tool","error":"missing field: tool arguments"}
498{"event":"turn_complete","turn_id":"turn-secret-1","status":"completed"}
499{"event":"turn_started","turn_id":"turn-secret-2"}
500not json at all
501"#;
502
503 let summary = analyze_session_failure_jsonl(jsonl);
504
505 assert_eq!(summary.malformed_lines, 1);
506 assert_eq!(summary.count(SessionFailureClass::MissingDependency), 1);
507 assert_eq!(summary.count(SessionFailureClass::CommandExit), 1);
508 assert_eq!(summary.count(SessionFailureClass::Network), 1);
509 assert_eq!(summary.count(SessionFailureClass::SandboxApproval), 1);
510 assert_eq!(summary.count(SessionFailureClass::Timeout), 1);
511 assert_eq!(summary.count(SessionFailureClass::BackgroundJob), 1);
512 assert_eq!(summary.count(SessionFailureClass::ToolSchema), 1);
513 assert_eq!(summary.count(SessionFailureClass::UnclosedTurn), 1);
514
515 let sources = summary
516 .sources
517 .get(&SessionFailureClass::MissingDependency)
518 .expect("missing-dependency source");
519 assert_eq!(sources[0].tool_name.as_deref(), Some("exec_shell"));
520 assert!(
521 sources[0]
522 .turn_ref
523 .as_deref()
524 .is_some_and(|turn| turn.starts_with("<redacted:")),
525 "turn ids must be redacted: {sources:?}"
526 );
527 }
528
529 #[test]
530 fn redacted_summary_omits_raw_messages_and_paths() {
531 let jsonl = r#"
532{"event":"turn_started","turn_id":"turn-secret-1"}
533{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"read_file","timestamp":"2026-06-25T12:34:56Z","error":"No such file or directory: /Users/alice/secret/project/.env"}
534"#;
535
536 let summary = analyze_session_failure_jsonl(jsonl);
537 let rendered = format_redacted_failure_summary(&summary);
538
539 assert!(rendered.contains("missing_dependency"));
540 assert!(rendered.contains("line 3"));
541 assert!(rendered.contains("tool=read_file"));
542 assert!(rendered.contains("ts=2026-06-25T12:34:56Z"));
543 assert!(!rendered.contains("alice"));
544 assert!(!rendered.contains(".env"));
545 assert!(!rendered.contains("turn-secret-1"));
546 }
547
548 #[test]
549 fn successful_content_does_not_create_unknown_failure() {
550 let jsonl = r#"
551{"event":"turn_started","turn_id":"turn-secret-1"}
552{"event":"tool_call_complete","turn_id":"turn-secret-1","tool_name":"exec_shell","success":true,"content":"command output mentioning error budgets is still normal content"}
553{"event":"turn_complete","turn_id":"turn-secret-1","status":"completed","message":"done"}
554"#;
555
556 let summary = analyze_session_failure_jsonl(jsonl);
557
558 assert_eq!(summary.count(SessionFailureClass::Unknown), 0);
559 assert_eq!(summary.count(SessionFailureClass::CommandExit), 0);
560 assert_eq!(summary.count(SessionFailureClass::UnclosedTurn), 0);
561 assert!(
562 summary.counts.is_empty(),
563 "summary should be empty: {summary:?}"
564 );
565 }
566
567 #[test]
568 fn empty_error_field_is_not_a_failure_signal() {
569 let jsonl = r#"
570{"event":"tool_call_complete","success":true,"error":"","stderr":""}
571"#;
572
573 let summary = analyze_session_failure_jsonl(jsonl);
574
575 assert!(
576 summary.counts.is_empty(),
577 "empty error/stderr sentinels should not signal failure: {summary:?}"
578 );
579 }
580
581 #[test]
582 fn nested_generic_fields_do_not_shadow_source_handles() {
583 let jsonl = r#"
584{"event":"tool_call_complete","turn_id":"turn-real","tool_name":"exec_shell","success":false,"payload":{"id":"nested-id","name":"nested-name","error":"nested error"}}
585"#;
586
587 let summary = analyze_session_failure_jsonl(jsonl);
588 let source = summary
589 .sources
590 .values()
591 .flat_map(|sources| sources.iter())
592 .next()
593 .expect("failure source");
594
595 assert_eq!(source.tool_name.as_deref(), Some("exec_shell"));
596 assert!(
597 source
598 .turn_ref
599 .as_deref()
600 .is_some_and(|turn| turn.starts_with("<redacted:")),
601 "top-level turn id should be redacted and used: {source:?}"
602 );
603 }
604
605 #[test]
606 fn deeply_nested_error_field_is_ignored() {
607 let jsonl = r#"
608{"event":"tool_call_complete","payload":{"a":{"b":{"c":{"d":{"e":{"error":"too deep"}}}}}}}
609"#;
610
611 let summary = analyze_session_failure_jsonl(jsonl);
612
613 assert!(
614 summary.counts.is_empty(),
615 "deeply nested error fields should not signal failure: {summary:?}"
616 );
617 }
618
619 #[test]
620 fn authentication_failures_are_model_failures_not_sandbox_approval() {
621 let jsonl = r#"
622{"event":"model_response","success":false,"error":"Authentication failed: invalid API key"}
623"#;
624
625 let summary = analyze_session_failure_jsonl(jsonl);
626
627 assert_eq!(summary.count(SessionFailureClass::Model), 1);
628 assert_eq!(summary.count(SessionFailureClass::SandboxApproval), 0);
629 }
630}