1use super::*;
2
3#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
4#[serde(tag = "type", rename_all = "snake_case")]
5pub enum ControlTraceDetail {
6 RequestCompleted {
7 method: Option<String>,
8 path: Option<String>,
9 status_code: Option<u16>,
10 duration_ms: Option<u64>,
11 station_name: Option<String>,
12 provider_id: Option<String>,
13 upstream_base_url: Option<String>,
14 service_tier: ServiceTierLog,
15 },
16 RetryOptions {
17 upstream_max_attempts: Option<u32>,
18 provider_max_attempts: Option<u32>,
19 allow_cross_station_before_first_output: Option<bool>,
20 },
21 AttemptSelect {
22 station_name: Option<String>,
23 upstream_index: Option<u64>,
24 upstream_base_url: Option<String>,
25 provider_id: Option<String>,
26 endpoint_id: Option<String>,
27 provider_endpoint_key: Option<String>,
28 preference_group: Option<u64>,
29 model: Option<String>,
30 },
31 LoadBalancerSelection {
32 mode: Option<String>,
33 pinned_source: Option<String>,
34 pinned_name: Option<String>,
35 selected_station: Option<String>,
36 selected_stations: Vec<String>,
37 active_station: Option<String>,
38 note: Option<String>,
39 },
40 ProviderRuntimeOverride {
41 provider_name: Option<String>,
42 endpoint_name: Option<String>,
43 base_urls: Vec<String>,
44 enabled: Option<bool>,
45 clear_enabled: bool,
46 runtime_state: Option<String>,
47 clear_runtime_state: bool,
48 },
49 RouteExecutorShadowMismatch {
50 request_model: Option<String>,
51 legacy_attempt_count: usize,
52 executor_attempt_count: usize,
53 first_mismatch_index: Option<usize>,
54 legacy_station_name: Option<String>,
55 legacy_upstream_index: Option<u64>,
56 legacy_provider_id: Option<String>,
57 executor_station_name: Option<String>,
58 executor_upstream_index: Option<u64>,
59 executor_provider_id: Option<String>,
60 },
61 RouteGraphSelectionExplain {
62 request_model: Option<String>,
63 continuity_class: Option<String>,
64 provider_failover_allowed: Option<bool>,
65 provider_failover_blocked_reason: Option<String>,
66 balance_signal_authoritative: Option<bool>,
67 affinity_policy: Option<String>,
68 affinity_source: Option<String>,
69 affinity_provider_endpoint_key: Option<String>,
70 selected_matches_affinity: Option<bool>,
71 selected_provider_id: Option<String>,
72 selected_endpoint_id: Option<String>,
73 selected_provider_endpoint_key: Option<String>,
74 selected_preference_group: Option<u64>,
75 skipped_higher_priority_groups: Vec<u64>,
76 skipped_higher_priority_candidates: Vec<ControlTraceRouteGraphSkippedCandidate>,
77 },
78 RetryEvent {
79 event_name: String,
80 station_name: Option<String>,
81 upstream_base_url: Option<String>,
82 mode: Option<String>,
83 note: Option<String>,
84 },
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
88pub struct ControlTraceRouteGraphSkippedCandidate {
89 pub provider_id: Option<String>,
90 pub endpoint_id: Option<String>,
91 pub provider_endpoint_key: Option<String>,
92 pub preference_group: Option<u64>,
93 pub route_path: Vec<String>,
94 pub reasons: Vec<String>,
95}
96
97#[derive(Debug, Clone, Serialize, Deserialize)]
98pub struct ControlTraceLogEntry {
99 pub ts_ms: u64,
100 pub kind: String,
101 #[serde(skip_serializing_if = "Option::is_none")]
102 pub service: Option<String>,
103 #[serde(skip_serializing_if = "Option::is_none")]
104 pub request_id: Option<u64>,
105 #[serde(default, skip_serializing_if = "Option::is_none")]
106 pub trace_id: Option<String>,
107 #[serde(skip_serializing_if = "Option::is_none")]
108 pub event: Option<String>,
109 #[serde(default, skip_serializing_if = "Option::is_none")]
110 pub detail: Option<ControlTraceDetail>,
111 #[serde(default)]
112 pub payload: JsonValue,
113}
114
115impl ControlTraceLogEntry {
116 pub fn resolved_trace_id(&self) -> Option<String> {
117 derive_control_trace_id(
118 self.trace_id.as_deref(),
119 self.service.as_deref(),
120 self.request_id,
121 &self.payload,
122 )
123 }
124
125 pub fn resolved_detail(&self) -> Option<ControlTraceDetail> {
126 self.detail.clone().or_else(|| {
127 infer_control_trace_detail(self.kind.as_str(), self.event.as_deref(), &self.payload)
128 })
129 }
130}
131
132pub fn control_trace_path() -> PathBuf {
133 std::env::var("CODEX_HELPER_CONTROL_TRACE_PATH")
134 .ok()
135 .map(|s| s.trim().to_string())
136 .filter(|s| !s.is_empty())
137 .map(PathBuf::from)
138 .unwrap_or_else(|| proxy_home_dir().join("logs").join("control_trace.jsonl"))
139}
140
141fn control_trace_enabled() -> bool {
142 static ENABLED: OnceLock<bool> = OnceLock::new();
143 *ENABLED.get_or_init(|| env_bool_default("CODEX_HELPER_CONTROL_TRACE", true))
144}
145
146fn retry_trace_enabled() -> bool {
147 static ENABLED: OnceLock<bool> = OnceLock::new();
148 *ENABLED.get_or_init(|| env_bool("CODEX_HELPER_RETRY_TRACE"))
149}
150
151fn retry_trace_path() -> PathBuf {
152 std::env::var("CODEX_HELPER_RETRY_TRACE_PATH")
153 .ok()
154 .map(|s| s.trim().to_string())
155 .filter(|s| !s.is_empty())
156 .map(PathBuf::from)
157 .unwrap_or_else(|| proxy_home_dir().join("logs").join("retry_trace.jsonl"))
158}
159
160fn control_trace_read_window(limit: usize) -> usize {
161 limit.saturating_mul(4).clamp(80, 400)
162}
163
164pub fn read_recent_control_trace_entries(
165 limit: usize,
166) -> anyhow::Result<Vec<ControlTraceLogEntry>> {
167 read_recent_control_trace_entries_from_path(
168 &control_trace_path(),
169 limit,
170 request_log_retention(),
171 )
172}
173
174fn read_recent_control_trace_entries_from_path(
175 path: &std::path::Path,
176 limit: usize,
177 retention: LogRetention,
178) -> anyhow::Result<Vec<ControlTraceLogEntry>> {
179 use std::collections::VecDeque;
180 use std::io::{BufRead, BufReader};
181
182 crate::local_log_store::repair_log(path, retention);
183 if !path.exists() {
184 return Ok(Vec::new());
185 }
186
187 let file = fs::File::open(path)?;
188 let reader = BufReader::new(file);
189 let mut ring = VecDeque::with_capacity(control_trace_read_window(limit));
190 for line in reader.lines() {
191 let line = line?;
192 if line.trim().is_empty() {
193 continue;
194 }
195 if ring.len() == ring.capacity() {
196 ring.pop_front();
197 }
198 ring.push_back(line);
199 }
200
201 let mut out = Vec::new();
202 for line in ring {
203 let Ok(entry) = serde_json::from_str::<ControlTraceLogEntry>(&line) else {
204 continue;
205 };
206 out.push(hydrate_control_trace_entry(entry));
207 }
208 out.sort_by_key(|entry| std::cmp::Reverse(entry.ts_ms));
209 Ok(out)
210}
211
212fn hydrate_control_trace_entry(mut entry: ControlTraceLogEntry) -> ControlTraceLogEntry {
213 if entry.trace_id.is_none() {
214 entry.trace_id = entry.resolved_trace_id();
215 }
216 if entry.detail.is_none() {
217 entry.detail =
218 infer_control_trace_detail(entry.kind.as_str(), entry.event.as_deref(), &entry.payload);
219 }
220 entry
221}
222
223fn json_u64_field(value: &JsonValue, key: &str) -> Option<u64> {
224 value.get(key).and_then(|value| match value {
225 JsonValue::Number(number) => number.as_u64(),
226 JsonValue::String(text) => text.trim().parse::<u64>().ok(),
227 _ => None,
228 })
229}
230
231fn json_u16_field(value: &JsonValue, key: &str) -> Option<u16> {
232 json_u64_field(value, key).and_then(|value| u16::try_from(value).ok())
233}
234
235fn json_string_field(value: &JsonValue, key: &str) -> Option<String> {
236 value
237 .get(key)
238 .and_then(|value| value.as_str())
239 .map(str::trim)
240 .filter(|value| !value.is_empty())
241 .map(ToOwned::to_owned)
242}
243
244fn json_bool_field(value: &JsonValue, key: &str) -> Option<bool> {
245 value.get(key).and_then(|value| value.as_bool())
246}
247
248fn json_nested_string_field(value: &JsonValue, path: &[&str]) -> Option<String> {
249 let mut current = value;
250 for segment in path {
251 current = current.get(*segment)?;
252 }
253 current
254 .as_str()
255 .map(str::trim)
256 .filter(|value| !value.is_empty())
257 .map(ToOwned::to_owned)
258}
259
260fn json_nested_u64_field(value: &JsonValue, path: &[&str]) -> Option<u64> {
261 let mut current = value;
262 for segment in path {
263 current = current.get(*segment)?;
264 }
265 match current {
266 JsonValue::Number(number) => number.as_u64(),
267 JsonValue::String(text) => text.trim().parse::<u64>().ok(),
268 _ => None,
269 }
270}
271
272fn json_string_vec_field(value: &JsonValue, key: &str) -> Vec<String> {
273 value
274 .get(key)
275 .and_then(|value| value.as_array())
276 .map(|items| {
277 items
278 .iter()
279 .filter_map(|item| item.as_str())
280 .map(str::trim)
281 .filter(|item| !item.is_empty())
282 .map(ToOwned::to_owned)
283 .collect::<Vec<_>>()
284 })
285 .unwrap_or_default()
286}
287
288fn json_u64_vec_field(value: &JsonValue, key: &str) -> Vec<u64> {
289 value
290 .get(key)
291 .and_then(|value| value.as_array())
292 .map(|items| {
293 items
294 .iter()
295 .filter_map(|item| match item {
296 JsonValue::Number(number) => number.as_u64(),
297 JsonValue::String(text) => text.trim().parse::<u64>().ok(),
298 _ => None,
299 })
300 .collect::<Vec<_>>()
301 })
302 .unwrap_or_default()
303}
304
305fn json_array_field<'a>(value: &'a JsonValue, key: &str) -> &'a [JsonValue] {
306 value
307 .get(key)
308 .and_then(|value| value.as_array())
309 .map(Vec::as_slice)
310 .unwrap_or(&[])
311}
312
313fn derive_control_trace_id(
314 trace_id: Option<&str>,
315 service: Option<&str>,
316 request_id: Option<u64>,
317 payload: &JsonValue,
318) -> Option<String> {
319 trace_id
320 .map(str::trim)
321 .filter(|value| !value.is_empty())
322 .map(ToOwned::to_owned)
323 .or_else(|| json_string_field(payload, "trace_id"))
324 .or_else(|| {
325 let request_id = request_id.or_else(|| json_u64_field(payload, "request_id"))?;
326 let service = service
327 .map(str::to_string)
328 .or_else(|| json_string_field(payload, "service"))
329 .unwrap_or_default();
330 Some(request_trace_id(service.as_str(), request_id))
331 })
332}
333
334fn json_service_tier_field(value: &JsonValue) -> ServiceTierLog {
335 value
336 .get("service_tier")
337 .cloned()
338 .and_then(|value| serde_json::from_value::<ServiceTierLog>(value).ok())
339 .unwrap_or_default()
340}
341
342fn infer_control_trace_detail(
343 kind: &str,
344 event: Option<&str>,
345 payload: &JsonValue,
346) -> Option<ControlTraceDetail> {
347 match kind {
348 "request_completed" => Some(ControlTraceDetail::RequestCompleted {
349 method: json_string_field(payload, "method"),
350 path: json_string_field(payload, "path"),
351 status_code: json_u16_field(payload, "status_code"),
352 duration_ms: json_u64_field(payload, "duration_ms"),
353 station_name: json_string_field(payload, "station_name"),
354 provider_id: json_string_field(payload, "provider_id"),
355 upstream_base_url: json_string_field(payload, "upstream_base_url"),
356 service_tier: json_service_tier_field(payload),
357 }),
358 "retry_trace" => infer_retry_control_trace_detail(event, payload),
359 _ => None,
360 }
361}
362
363fn infer_retry_control_trace_detail(
364 event: Option<&str>,
365 payload: &JsonValue,
366) -> Option<ControlTraceDetail> {
367 let event_name = event
368 .map(str::to_string)
369 .or_else(|| json_string_field(payload, "event"))
370 .unwrap_or_else(|| "retry_trace".to_string());
371
372 match event_name.as_str() {
373 "attempt_select" => Some(ControlTraceDetail::AttemptSelect {
374 station_name: json_nested_string_field(payload, &["compatibility", "station_name"])
375 .or_else(|| json_string_field(payload, "station_name")),
376 upstream_index: json_nested_u64_field(payload, &["compatibility", "upstream_index"])
377 .or_else(|| json_u64_field(payload, "upstream_index")),
378 upstream_base_url: json_string_field(payload, "upstream_base_url"),
379 provider_id: json_string_field(payload, "provider_id"),
380 endpoint_id: json_string_field(payload, "endpoint_id"),
381 provider_endpoint_key: json_string_field(payload, "provider_endpoint_key"),
382 preference_group: json_u64_field(payload, "preference_group"),
383 model: json_string_field(payload, "model"),
384 }),
385 "retry_options" => Some(ControlTraceDetail::RetryOptions {
386 upstream_max_attempts: json_nested_u64_field(payload, &["upstream", "max_attempts"])
387 .and_then(|value| u32::try_from(value).ok()),
388 provider_max_attempts: json_nested_u64_field(payload, &["provider", "max_attempts"])
389 .and_then(|value| u32::try_from(value).ok()),
390 allow_cross_station_before_first_output: json_bool_field(
391 payload,
392 "allow_cross_station_before_first_output",
393 ),
394 }),
395 "lbs_for_request" => Some(ControlTraceDetail::LoadBalancerSelection {
396 mode: json_string_field(payload, "mode"),
397 pinned_source: json_string_field(payload, "pinned_source"),
398 pinned_name: json_string_field(payload, "pinned_name"),
399 selected_station: json_string_field(payload, "selected_station"),
400 selected_stations: json_string_vec_field(payload, "selected_stations"),
401 active_station: json_string_field(payload, "active_station"),
402 note: json_string_field(payload, "note"),
403 }),
404 "provider_runtime_override" => Some(ControlTraceDetail::ProviderRuntimeOverride {
405 provider_name: json_string_field(payload, "provider_name"),
406 endpoint_name: json_string_field(payload, "endpoint_name"),
407 base_urls: json_string_vec_field(payload, "base_urls"),
408 enabled: json_bool_field(payload, "enabled"),
409 clear_enabled: json_bool_field(payload, "clear_enabled").unwrap_or(false),
410 runtime_state: json_string_field(payload, "runtime_state"),
411 clear_runtime_state: json_bool_field(payload, "clear_runtime_state").unwrap_or(false),
412 }),
413 "route_executor_shadow_mismatch" => Some(route_executor_shadow_mismatch_detail(payload)),
414 "route_graph_selection_explain" => Some(route_graph_selection_explain_detail(payload)),
415 _ => Some(ControlTraceDetail::RetryEvent {
416 event_name,
417 station_name: json_string_field(payload, "station_name")
418 .or_else(|| json_string_field(payload, "selected_station")),
419 upstream_base_url: json_string_field(payload, "upstream_base_url"),
420 mode: json_string_field(payload, "mode"),
421 note: json_string_field(payload, "note"),
422 }),
423 }
424}
425
426fn route_graph_selection_explain_detail(payload: &JsonValue) -> ControlTraceDetail {
427 ControlTraceDetail::RouteGraphSelectionExplain {
428 request_model: json_string_field(payload, "request_model"),
429 continuity_class: json_nested_string_field(payload, &["continuity", "class"]),
430 provider_failover_allowed: payload
431 .get("continuity")
432 .and_then(|continuity| json_bool_field(continuity, "provider_failover_allowed")),
433 provider_failover_blocked_reason: json_nested_string_field(
434 payload,
435 &["continuity", "provider_failover_blocked_reason"],
436 ),
437 balance_signal_authoritative: payload
438 .get("continuity")
439 .and_then(|continuity| json_bool_field(continuity, "balance_signal_authoritative")),
440 affinity_policy: json_nested_string_field(payload, &["affinity", "policy"]),
441 affinity_source: json_nested_string_field(payload, &["affinity", "source"]),
442 affinity_provider_endpoint_key: json_nested_string_field(
443 payload,
444 &["affinity", "provider_endpoint_key"],
445 ),
446 selected_matches_affinity: payload
447 .get("affinity")
448 .and_then(|affinity| json_bool_field(affinity, "selected_matches_affinity")),
449 selected_provider_id: json_nested_string_field(payload, &["selected", "provider_id"]),
450 selected_endpoint_id: json_nested_string_field(payload, &["selected", "endpoint_id"]),
451 selected_provider_endpoint_key: json_nested_string_field(
452 payload,
453 &["selected", "provider_endpoint_key"],
454 ),
455 selected_preference_group: json_nested_u64_field(
456 payload,
457 &["selected", "preference_group"],
458 ),
459 skipped_higher_priority_groups: json_u64_vec_field(
460 payload,
461 "skipped_higher_priority_groups",
462 ),
463 skipped_higher_priority_candidates: json_array_field(
464 payload,
465 "skipped_higher_priority_candidates",
466 )
467 .iter()
468 .map(|candidate| ControlTraceRouteGraphSkippedCandidate {
469 provider_id: json_string_field(candidate, "provider_id"),
470 endpoint_id: json_string_field(candidate, "endpoint_id"),
471 provider_endpoint_key: json_string_field(candidate, "provider_endpoint_key"),
472 preference_group: json_u64_field(candidate, "preference_group"),
473 route_path: json_string_vec_field(candidate, "route_path"),
474 reasons: json_string_vec_field(candidate, "reasons"),
475 })
476 .collect(),
477 }
478}
479
480fn route_executor_shadow_mismatch_detail(payload: &JsonValue) -> ControlTraceDetail {
481 let legacy_attempts = json_array_field(payload, "legacy_attempts");
482 let executor_attempts = json_array_field(payload, "executor_attempts");
483 let first_mismatch_index = first_mismatch_index(legacy_attempts, executor_attempts);
484 let legacy_attempt = first_mismatch_index.and_then(|idx| legacy_attempts.get(idx));
485 let executor_attempt = first_mismatch_index.and_then(|idx| executor_attempts.get(idx));
486
487 ControlTraceDetail::RouteExecutorShadowMismatch {
488 request_model: json_string_field(payload, "request_model"),
489 legacy_attempt_count: legacy_attempts.len(),
490 executor_attempt_count: executor_attempts.len(),
491 first_mismatch_index,
492 legacy_station_name: legacy_attempt
493 .and_then(|attempt| json_string_field(attempt, "station_name")),
494 legacy_upstream_index: legacy_attempt
495 .and_then(|attempt| json_u64_field(attempt, "upstream_index")),
496 legacy_provider_id: legacy_attempt
497 .and_then(|attempt| json_string_field(attempt, "provider_id")),
498 executor_station_name: executor_attempt
499 .and_then(|attempt| json_string_field(attempt, "station_name")),
500 executor_upstream_index: executor_attempt
501 .and_then(|attempt| json_u64_field(attempt, "upstream_index")),
502 executor_provider_id: executor_attempt
503 .and_then(|attempt| json_string_field(attempt, "provider_id")),
504 }
505}
506
507fn first_mismatch_index(left: &[JsonValue], right: &[JsonValue]) -> Option<usize> {
508 let shared_len = left.len().min(right.len());
509 for idx in 0..shared_len {
510 if left[idx] != right[idx] {
511 return Some(idx);
512 }
513 }
514 (left.len() != right.len()).then_some(shared_len)
515}
516
517pub(super) fn append_control_trace_payload(
518 opt: RequestLogOptions,
519 kind: &'static str,
520 service: Option<&str>,
521 request_id: Option<u64>,
522 event: Option<&str>,
523 ts_ms: u64,
524 payload: JsonValue,
525) {
526 if !control_trace_enabled() {
527 return;
528 }
529 let path = control_trace_path();
530 let entry = make_control_trace_entry(kind, service, request_id, event, ts_ms, payload);
531 if let Ok(line) = serde_json::to_string(&entry) {
532 let _ = append_json_line(&path, opt, &line);
533 }
534}
535
536fn make_control_trace_entry(
537 kind: &'static str,
538 service: Option<&str>,
539 request_id: Option<u64>,
540 event: Option<&str>,
541 ts_ms: u64,
542 payload: JsonValue,
543) -> ControlTraceLogEntry {
544 let detail = infer_control_trace_detail(kind, event, &payload);
545 let trace_id = derive_control_trace_id(None, service, request_id, &payload);
546 ControlTraceLogEntry {
547 ts_ms,
548 kind: kind.to_string(),
549 service: service.map(str::to_string),
550 request_id,
551 trace_id,
552 event: event.map(str::to_string),
553 detail,
554 payload,
555 }
556}
557
558pub fn log_retry_trace(mut event: JsonValue) {
559 let legacy_enabled = retry_trace_enabled();
560 let unified_enabled = control_trace_enabled();
561 if !legacy_enabled && !unified_enabled {
562 return;
563 }
564
565 if let JsonValue::Object(ref mut obj) = event {
566 obj.entry("ts_ms".to_string())
567 .or_insert_with(|| JsonValue::Number(serde_json::Number::from(now_ms())));
568 }
569
570 let ts_ms = json_u64_field(&event, "ts_ms").unwrap_or_else(now_ms);
571 let service = json_string_field(&event, "service");
572 let request_id = json_u64_field(&event, "request_id");
573 let event_name = json_string_field(&event, "event");
574 let opt = request_log_options();
575 let _guard = match log_lock().lock() {
576 Ok(g) => g,
577 Err(e) => e.into_inner(),
578 };
579
580 if legacy_enabled {
581 let path = retry_trace_path();
582 if let Ok(line) = serde_json::to_string(&event) {
583 let _ = append_json_line(&path, opt, &line);
584 }
585 }
586
587 append_control_trace_payload(
588 opt,
589 "retry_trace",
590 service.as_deref(),
591 request_id,
592 event_name.as_deref(),
593 ts_ms,
594 event,
595 );
596}
597
598#[cfg(test)]
599mod tests {
600 use super::*;
601
602 fn temp_control_trace_path(test_name: &str) -> std::path::PathBuf {
603 let path = std::env::temp_dir().join(format!(
604 "codex-helper-control-trace-{test_name}-{}-{}.jsonl",
605 std::process::id(),
606 now_ms()
607 ));
608 let _ = std::fs::remove_file(&path);
609 path
610 }
611
612 #[test]
613 fn json_field_helpers_extract_string_and_numeric_values() {
614 let event = serde_json::json!({
615 "service": "codex",
616 "request_id": "42",
617 "ts_ms": 99,
618 });
619
620 assert_eq!(
621 json_string_field(&event, "service").as_deref(),
622 Some("codex")
623 );
624 assert_eq!(json_u64_field(&event, "request_id"), Some(42));
625 assert_eq!(json_u64_field(&event, "ts_ms"), Some(99));
626 }
627
628 #[test]
629 fn make_control_trace_entry_keeps_kind_event_and_request_id() {
630 let entry = make_control_trace_entry(
631 "retry_trace",
632 Some("codex"),
633 Some(7),
634 Some("attempt_select"),
635 123,
636 serde_json::json!({
637 "event": "attempt_select",
638 "service": "codex",
639 "request_id": 7,
640 "provider_id": "monthly",
641 "endpoint_id": "default",
642 "provider_endpoint_key": "codex/monthly/default",
643 "preference_group": 0,
644 "compatibility": {
645 "station_name": "routing",
646 "upstream_index": 0
647 },
648 }),
649 );
650
651 let value = serde_json::to_value(entry).expect("serialize control trace entry");
652 assert_eq!(value["kind"].as_str(), Some("retry_trace"));
653 assert_eq!(value["event"].as_str(), Some("attempt_select"));
654 assert_eq!(value["request_id"].as_u64(), Some(7));
655 assert_eq!(value["trace_id"].as_str(), Some("codex-7"));
656 assert_eq!(value["service"].as_str(), Some("codex"));
657 assert_eq!(value["payload"]["event"].as_str(), Some("attempt_select"));
658 assert_eq!(value["detail"]["type"].as_str(), Some("attempt_select"));
659 assert_eq!(
660 value["detail"]["provider_endpoint_key"].as_str(),
661 Some("codex/monthly/default")
662 );
663 assert_eq!(value["detail"]["preference_group"].as_u64(), Some(0));
664 assert_eq!(value["detail"]["station_name"].as_str(), Some("routing"));
665 assert_eq!(value["detail"]["upstream_index"].as_u64(), Some(0));
666 }
667
668 #[test]
669 fn hydrate_control_trace_entry_adds_trace_id_to_legacy_event() {
670 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
671 "ts_ms": 1,
672 "kind": "retry_trace",
673 "service": "codex",
674 "request_id": 7,
675 "event": "attempt_select",
676 "payload": {
677 "event": "attempt_select",
678 "station_name": "right"
679 }
680 }))
681 .expect("deserialize legacy control trace");
682
683 let entry = hydrate_control_trace_entry(entry);
684
685 assert_eq!(entry.trace_id.as_deref(), Some("codex-7"));
686 assert_eq!(
687 entry.detail,
688 Some(ControlTraceDetail::AttemptSelect {
689 station_name: Some("right".to_string()),
690 upstream_index: None,
691 upstream_base_url: None,
692 provider_id: None,
693 endpoint_id: None,
694 provider_endpoint_key: None,
695 preference_group: None,
696 model: None,
697 })
698 );
699 }
700
701 #[test]
702 fn read_recent_control_trace_repairs_oversized_active_log_before_scan() {
703 let path = temp_control_trace_path("read-repair");
704 std::fs::write(&path, vec![b'x'; 32]).expect("seed oversized control trace");
705
706 let entries =
707 read_recent_control_trace_entries_from_path(&path, 10, LogRetention::new(16, 1))
708 .expect("read repaired control trace");
709
710 assert!(entries.is_empty());
711 assert!(
712 !path.exists(),
713 "oversized active control trace should be rotated away before reading"
714 );
715 assert!(
716 crate::local_log_store::collect_rotated_logs(&path).is_empty(),
717 "oversized rotated control trace should be pruned by retention budget"
718 );
719 let _ = std::fs::remove_file(path);
720 }
721
722 #[test]
723 fn resolved_trace_id_prefers_payload_trace_id() {
724 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
725 "ts_ms": 1,
726 "kind": "retry_trace",
727 "service": "codex",
728 "request_id": 7,
729 "payload": {
730 "trace_id": "external-123",
731 "service": "codex",
732 "request_id": 8
733 }
734 }))
735 .expect("deserialize control trace");
736
737 assert_eq!(entry.resolved_trace_id().as_deref(), Some("external-123"));
738 }
739
740 #[test]
741 fn control_trace_entry_resolved_detail_infers_request_completed_from_legacy_payload() {
742 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
743 "ts_ms": 1,
744 "kind": "request_completed",
745 "service": "codex",
746 "request_id": 11,
747 "event": "request_completed",
748 "payload": {
749 "method": "POST",
750 "path": "/v1/responses",
751 "status_code": 200,
752 "duration_ms": 321,
753 "station_name": "right",
754 "provider_id": "right",
755 "service_tier": {
756 "effective": "priority"
757 }
758 }
759 }))
760 .expect("deserialize control trace");
761
762 assert_eq!(
763 entry.resolved_detail(),
764 Some(ControlTraceDetail::RequestCompleted {
765 method: Some("POST".to_string()),
766 path: Some("/v1/responses".to_string()),
767 status_code: Some(200),
768 duration_ms: Some(321),
769 station_name: Some("right".to_string()),
770 provider_id: Some("right".to_string()),
771 upstream_base_url: None,
772 service_tier: ServiceTierLog {
773 requested: None,
774 effective: Some("priority".to_string()),
775 actual: None,
776 },
777 })
778 );
779 }
780
781 #[test]
782 fn control_trace_entry_resolved_detail_infers_provider_runtime_override() {
783 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
784 "ts_ms": 1,
785 "kind": "retry_trace",
786 "service": "codex",
787 "event": "provider_runtime_override",
788 "payload": {
789 "event": "provider_runtime_override",
790 "provider_name": "alpha",
791 "endpoint_name": "default",
792 "base_urls": ["https://alpha.example/v1"],
793 "enabled": false,
794 "clear_enabled": false,
795 "runtime_state": "breaker_open",
796 "clear_runtime_state": false
797 }
798 }))
799 .expect("deserialize provider runtime trace");
800
801 assert_eq!(
802 entry.resolved_detail(),
803 Some(ControlTraceDetail::ProviderRuntimeOverride {
804 provider_name: Some("alpha".to_string()),
805 endpoint_name: Some("default".to_string()),
806 base_urls: vec!["https://alpha.example/v1".to_string()],
807 enabled: Some(false),
808 clear_enabled: false,
809 runtime_state: Some("breaker_open".to_string()),
810 clear_runtime_state: false,
811 })
812 );
813 }
814
815 #[test]
816 fn control_trace_entry_resolved_detail_infers_route_graph_selection_explain() {
817 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
818 "ts_ms": 1,
819 "kind": "retry_trace",
820 "service": "codex",
821 "request_id": 42,
822 "event": "route_graph_selection_explain",
823 "payload": {
824 "event": "route_graph_selection_explain",
825 "service": "codex",
826 "request_id": 42,
827 "request_model": "gpt-5.4",
828 "continuity": {
829 "class": "stateless_or_session_preferred",
830 "provider_failover_allowed": true,
831 "provider_failover_blocked_reason": null,
832 "balance_signal_authoritative": false
833 },
834 "affinity": {
835 "policy": "preferred_group",
836 "provider_endpoint_key": "codex/chili/default",
837 "source": "session_route_affinity",
838 "selected_matches_affinity": false
839 },
840 "selected": {
841 "provider_id": "chili",
842 "endpoint_id": "default",
843 "provider_endpoint_key": "codex/chili/default",
844 "preference_group": 1,
845 "route_path": ["entry", "fallback"]
846 },
847 "skipped_higher_priority_groups": [0],
848 "skipped_higher_priority_candidates": [
849 {
850 "provider_id": "monthly",
851 "endpoint_id": "default",
852 "provider_endpoint_key": "codex/monthly/default",
853 "preference_group": 0,
854 "route_path": ["entry", "monthly"],
855 "reasons": ["usage_exhausted"]
856 }
857 ]
858 }
859 }))
860 .expect("deserialize route graph selection explain trace");
861
862 assert_eq!(
863 entry.resolved_detail(),
864 Some(ControlTraceDetail::RouteGraphSelectionExplain {
865 request_model: Some("gpt-5.4".to_string()),
866 continuity_class: Some("stateless_or_session_preferred".to_string()),
867 provider_failover_allowed: Some(true),
868 provider_failover_blocked_reason: None,
869 balance_signal_authoritative: Some(false),
870 affinity_policy: Some("preferred_group".to_string()),
871 affinity_source: Some("session_route_affinity".to_string()),
872 affinity_provider_endpoint_key: Some("codex/chili/default".to_string()),
873 selected_matches_affinity: Some(false),
874 selected_provider_id: Some("chili".to_string()),
875 selected_endpoint_id: Some("default".to_string()),
876 selected_provider_endpoint_key: Some("codex/chili/default".to_string()),
877 selected_preference_group: Some(1),
878 skipped_higher_priority_groups: vec![0],
879 skipped_higher_priority_candidates: vec![ControlTraceRouteGraphSkippedCandidate {
880 provider_id: Some("monthly".to_string()),
881 endpoint_id: Some("default".to_string()),
882 provider_endpoint_key: Some("codex/monthly/default".to_string()),
883 preference_group: Some(0),
884 route_path: vec!["entry".to_string(), "monthly".to_string()],
885 reasons: vec!["usage_exhausted".to_string()],
886 }],
887 })
888 );
889 }
890
891 #[test]
892 fn control_trace_entry_resolved_detail_infers_route_executor_shadow_mismatch() {
893 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
894 "ts_ms": 1,
895 "kind": "retry_trace",
896 "service": "codex",
897 "request_id": 19,
898 "event": "route_executor_shadow_mismatch",
899 "payload": {
900 "event": "route_executor_shadow_mismatch",
901 "service": "codex",
902 "request_id": 19,
903 "request_model": "gpt-5",
904 "legacy_attempts": [
905 {
906 "decision": "selected",
907 "station_name": "routing",
908 "upstream_index": 1,
909 "upstream_base_url": "https://legacy.example/v1",
910 "provider_id": "legacy"
911 }
912 ],
913 "executor_attempts": [
914 {
915 "decision": "selected",
916 "station_name": "routing",
917 "upstream_index": 0,
918 "upstream_base_url": "https://executor.example/v1",
919 "provider_id": "executor"
920 },
921 {
922 "decision": "selected",
923 "station_name": "routing",
924 "upstream_index": 1,
925 "upstream_base_url": "https://legacy.example/v1",
926 "provider_id": "legacy"
927 }
928 ]
929 }
930 }))
931 .expect("deserialize shadow mismatch trace");
932
933 let entry = hydrate_control_trace_entry(entry);
934 assert_eq!(
935 entry.resolved_detail(),
936 Some(ControlTraceDetail::RouteExecutorShadowMismatch {
937 request_model: Some("gpt-5".to_string()),
938 legacy_attempt_count: 1,
939 executor_attempt_count: 2,
940 first_mismatch_index: Some(0),
941 legacy_station_name: Some("routing".to_string()),
942 legacy_upstream_index: Some(1),
943 legacy_provider_id: Some("legacy".to_string()),
944 executor_station_name: Some("routing".to_string()),
945 executor_upstream_index: Some(0),
946 executor_provider_id: Some("executor".to_string()),
947 })
948 );
949
950 let value = serde_json::to_value(entry).expect("serialize shadow mismatch trace");
951 assert_eq!(
952 value["detail"]["type"].as_str(),
953 Some("route_executor_shadow_mismatch")
954 );
955 }
956
957 #[test]
958 fn first_mismatch_index_reports_length_mismatch_after_shared_prefix() {
959 let left = vec![serde_json::json!({"provider_id": "a"})];
960 let right = vec![
961 serde_json::json!({"provider_id": "a"}),
962 serde_json::json!({"provider_id": "b"}),
963 ];
964
965 assert_eq!(first_mismatch_index(&left, &right), Some(1));
966 assert_eq!(first_mismatch_index(&left, &left), None);
967 }
968}