1use super::*;
2
3#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
4#[serde(tag = "type", rename_all = "snake_case")]
5pub enum ControlTraceDetail {
6 RequestCompleted {
7 method: Option<String>,
8 path: Option<String>,
9 status_code: Option<u16>,
10 duration_ms: Option<u64>,
11 station_name: Option<String>,
12 provider_id: Option<String>,
13 upstream_base_url: Option<String>,
14 service_tier: ServiceTierLog,
15 },
16 RetryOptions {
17 upstream_max_attempts: Option<u32>,
18 provider_max_attempts: Option<u32>,
19 allow_cross_station_before_first_output: Option<bool>,
20 },
21 AttemptSelect {
22 station_name: Option<String>,
23 upstream_index: Option<u64>,
24 upstream_base_url: Option<String>,
25 provider_id: Option<String>,
26 endpoint_id: Option<String>,
27 provider_endpoint_key: Option<String>,
28 preference_group: Option<u64>,
29 model: Option<String>,
30 },
31 LoadBalancerSelection {
32 mode: Option<String>,
33 pinned_source: Option<String>,
34 pinned_name: Option<String>,
35 selected_station: Option<String>,
36 selected_stations: Vec<String>,
37 active_station: Option<String>,
38 note: Option<String>,
39 },
40 ProviderRuntimeOverride {
41 provider_name: Option<String>,
42 endpoint_name: Option<String>,
43 base_urls: Vec<String>,
44 enabled: Option<bool>,
45 clear_enabled: bool,
46 runtime_state: Option<String>,
47 clear_runtime_state: bool,
48 },
49 RouteExecutorShadowMismatch {
50 request_model: Option<String>,
51 legacy_attempt_count: usize,
52 executor_attempt_count: usize,
53 first_mismatch_index: Option<usize>,
54 legacy_station_name: Option<String>,
55 legacy_upstream_index: Option<u64>,
56 legacy_provider_id: Option<String>,
57 executor_station_name: Option<String>,
58 executor_upstream_index: Option<u64>,
59 executor_provider_id: Option<String>,
60 },
61 RouteGraphSelectionExplain {
62 request_model: Option<String>,
63 continuity_class: Option<String>,
64 provider_failover_allowed: Option<bool>,
65 provider_failover_blocked_reason: Option<String>,
66 balance_signal_authoritative: Option<bool>,
67 affinity_policy: Option<String>,
68 affinity_source: Option<String>,
69 affinity_provider_endpoint_key: Option<String>,
70 selected_matches_affinity: Option<bool>,
71 selected_provider_id: Option<String>,
72 selected_endpoint_id: Option<String>,
73 selected_provider_endpoint_key: Option<String>,
74 selected_preference_group: Option<u64>,
75 skipped_higher_priority_groups: Vec<u64>,
76 skipped_higher_priority_candidates: Vec<ControlTraceRouteGraphSkippedCandidate>,
77 },
78 RetryEvent {
79 event_name: String,
80 station_name: Option<String>,
81 upstream_base_url: Option<String>,
82 mode: Option<String>,
83 note: Option<String>,
84 },
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
88pub struct ControlTraceRouteGraphSkippedCandidate {
89 pub provider_id: Option<String>,
90 pub endpoint_id: Option<String>,
91 pub provider_endpoint_key: Option<String>,
92 pub preference_group: Option<u64>,
93 pub route_path: Vec<String>,
94 pub reasons: Vec<String>,
95}
96
97#[derive(Debug, Clone, Serialize, Deserialize)]
98pub struct ControlTraceLogEntry {
99 pub ts_ms: u64,
100 pub kind: String,
101 #[serde(skip_serializing_if = "Option::is_none")]
102 pub service: Option<String>,
103 #[serde(skip_serializing_if = "Option::is_none")]
104 pub request_id: Option<u64>,
105 #[serde(default, skip_serializing_if = "Option::is_none")]
106 pub trace_id: Option<String>,
107 #[serde(skip_serializing_if = "Option::is_none")]
108 pub event: Option<String>,
109 #[serde(default, skip_serializing_if = "Option::is_none")]
110 pub detail: Option<ControlTraceDetail>,
111 #[serde(default)]
112 pub payload: JsonValue,
113}
114
115impl ControlTraceLogEntry {
116 pub fn resolved_trace_id(&self) -> Option<String> {
117 derive_control_trace_id(
118 self.trace_id.as_deref(),
119 self.service.as_deref(),
120 self.request_id,
121 &self.payload,
122 )
123 }
124
125 pub fn resolved_detail(&self) -> Option<ControlTraceDetail> {
126 self.detail.clone().or_else(|| {
127 infer_control_trace_detail(self.kind.as_str(), self.event.as_deref(), &self.payload)
128 })
129 }
130}
131
132pub fn control_trace_path() -> PathBuf {
133 std::env::var("CODEX_HELPER_CONTROL_TRACE_PATH")
134 .ok()
135 .map(|s| s.trim().to_string())
136 .filter(|s| !s.is_empty())
137 .map(PathBuf::from)
138 .unwrap_or_else(|| proxy_home_dir().join("logs").join("control_trace.jsonl"))
139}
140
141fn control_trace_enabled() -> bool {
142 static ENABLED: OnceLock<bool> = OnceLock::new();
143 *ENABLED.get_or_init(|| env_bool_default("CODEX_HELPER_CONTROL_TRACE", true))
144}
145
146fn retry_trace_enabled() -> bool {
147 static ENABLED: OnceLock<bool> = OnceLock::new();
148 *ENABLED.get_or_init(|| env_bool("CODEX_HELPER_RETRY_TRACE"))
149}
150
151fn retry_trace_path() -> PathBuf {
152 std::env::var("CODEX_HELPER_RETRY_TRACE_PATH")
153 .ok()
154 .map(|s| s.trim().to_string())
155 .filter(|s| !s.is_empty())
156 .map(PathBuf::from)
157 .unwrap_or_else(|| proxy_home_dir().join("logs").join("retry_trace.jsonl"))
158}
159
160fn control_trace_read_window(limit: usize) -> usize {
161 limit.saturating_mul(4).clamp(80, 400)
162}
163
164pub fn read_recent_control_trace_entries(
165 limit: usize,
166) -> anyhow::Result<Vec<ControlTraceLogEntry>> {
167 use std::collections::VecDeque;
168 use std::io::{BufRead, BufReader};
169
170 let path = control_trace_path();
171 if !path.exists() {
172 return Ok(Vec::new());
173 }
174
175 let file = fs::File::open(&path)?;
176 let reader = BufReader::new(file);
177 let mut ring = VecDeque::with_capacity(control_trace_read_window(limit));
178 for line in reader.lines() {
179 let line = line?;
180 if line.trim().is_empty() {
181 continue;
182 }
183 if ring.len() == ring.capacity() {
184 ring.pop_front();
185 }
186 ring.push_back(line);
187 }
188
189 let mut out = Vec::new();
190 for line in ring {
191 let Ok(entry) = serde_json::from_str::<ControlTraceLogEntry>(&line) else {
192 continue;
193 };
194 out.push(hydrate_control_trace_entry(entry));
195 }
196 out.sort_by_key(|entry| std::cmp::Reverse(entry.ts_ms));
197 Ok(out)
198}
199
200fn hydrate_control_trace_entry(mut entry: ControlTraceLogEntry) -> ControlTraceLogEntry {
201 if entry.trace_id.is_none() {
202 entry.trace_id = entry.resolved_trace_id();
203 }
204 if entry.detail.is_none() {
205 entry.detail =
206 infer_control_trace_detail(entry.kind.as_str(), entry.event.as_deref(), &entry.payload);
207 }
208 entry
209}
210
211fn json_u64_field(value: &JsonValue, key: &str) -> Option<u64> {
212 value.get(key).and_then(|value| match value {
213 JsonValue::Number(number) => number.as_u64(),
214 JsonValue::String(text) => text.trim().parse::<u64>().ok(),
215 _ => None,
216 })
217}
218
219fn json_u16_field(value: &JsonValue, key: &str) -> Option<u16> {
220 json_u64_field(value, key).and_then(|value| u16::try_from(value).ok())
221}
222
223fn json_string_field(value: &JsonValue, key: &str) -> Option<String> {
224 value
225 .get(key)
226 .and_then(|value| value.as_str())
227 .map(str::trim)
228 .filter(|value| !value.is_empty())
229 .map(ToOwned::to_owned)
230}
231
232fn json_bool_field(value: &JsonValue, key: &str) -> Option<bool> {
233 value.get(key).and_then(|value| value.as_bool())
234}
235
236fn json_nested_string_field(value: &JsonValue, path: &[&str]) -> Option<String> {
237 let mut current = value;
238 for segment in path {
239 current = current.get(*segment)?;
240 }
241 current
242 .as_str()
243 .map(str::trim)
244 .filter(|value| !value.is_empty())
245 .map(ToOwned::to_owned)
246}
247
248fn json_nested_u64_field(value: &JsonValue, path: &[&str]) -> Option<u64> {
249 let mut current = value;
250 for segment in path {
251 current = current.get(*segment)?;
252 }
253 match current {
254 JsonValue::Number(number) => number.as_u64(),
255 JsonValue::String(text) => text.trim().parse::<u64>().ok(),
256 _ => None,
257 }
258}
259
260fn json_string_vec_field(value: &JsonValue, key: &str) -> Vec<String> {
261 value
262 .get(key)
263 .and_then(|value| value.as_array())
264 .map(|items| {
265 items
266 .iter()
267 .filter_map(|item| item.as_str())
268 .map(str::trim)
269 .filter(|item| !item.is_empty())
270 .map(ToOwned::to_owned)
271 .collect::<Vec<_>>()
272 })
273 .unwrap_or_default()
274}
275
276fn json_u64_vec_field(value: &JsonValue, key: &str) -> Vec<u64> {
277 value
278 .get(key)
279 .and_then(|value| value.as_array())
280 .map(|items| {
281 items
282 .iter()
283 .filter_map(|item| match item {
284 JsonValue::Number(number) => number.as_u64(),
285 JsonValue::String(text) => text.trim().parse::<u64>().ok(),
286 _ => None,
287 })
288 .collect::<Vec<_>>()
289 })
290 .unwrap_or_default()
291}
292
293fn json_array_field<'a>(value: &'a JsonValue, key: &str) -> &'a [JsonValue] {
294 value
295 .get(key)
296 .and_then(|value| value.as_array())
297 .map(Vec::as_slice)
298 .unwrap_or(&[])
299}
300
301fn derive_control_trace_id(
302 trace_id: Option<&str>,
303 service: Option<&str>,
304 request_id: Option<u64>,
305 payload: &JsonValue,
306) -> Option<String> {
307 trace_id
308 .map(str::trim)
309 .filter(|value| !value.is_empty())
310 .map(ToOwned::to_owned)
311 .or_else(|| json_string_field(payload, "trace_id"))
312 .or_else(|| {
313 let request_id = request_id.or_else(|| json_u64_field(payload, "request_id"))?;
314 let service = service
315 .map(str::to_string)
316 .or_else(|| json_string_field(payload, "service"))
317 .unwrap_or_default();
318 Some(request_trace_id(service.as_str(), request_id))
319 })
320}
321
322fn json_service_tier_field(value: &JsonValue) -> ServiceTierLog {
323 value
324 .get("service_tier")
325 .cloned()
326 .and_then(|value| serde_json::from_value::<ServiceTierLog>(value).ok())
327 .unwrap_or_default()
328}
329
330fn infer_control_trace_detail(
331 kind: &str,
332 event: Option<&str>,
333 payload: &JsonValue,
334) -> Option<ControlTraceDetail> {
335 match kind {
336 "request_completed" => Some(ControlTraceDetail::RequestCompleted {
337 method: json_string_field(payload, "method"),
338 path: json_string_field(payload, "path"),
339 status_code: json_u16_field(payload, "status_code"),
340 duration_ms: json_u64_field(payload, "duration_ms"),
341 station_name: json_string_field(payload, "station_name"),
342 provider_id: json_string_field(payload, "provider_id"),
343 upstream_base_url: json_string_field(payload, "upstream_base_url"),
344 service_tier: json_service_tier_field(payload),
345 }),
346 "retry_trace" => infer_retry_control_trace_detail(event, payload),
347 _ => None,
348 }
349}
350
351fn infer_retry_control_trace_detail(
352 event: Option<&str>,
353 payload: &JsonValue,
354) -> Option<ControlTraceDetail> {
355 let event_name = event
356 .map(str::to_string)
357 .or_else(|| json_string_field(payload, "event"))
358 .unwrap_or_else(|| "retry_trace".to_string());
359
360 match event_name.as_str() {
361 "attempt_select" => Some(ControlTraceDetail::AttemptSelect {
362 station_name: json_nested_string_field(payload, &["compatibility", "station_name"])
363 .or_else(|| json_string_field(payload, "station_name")),
364 upstream_index: json_nested_u64_field(payload, &["compatibility", "upstream_index"])
365 .or_else(|| json_u64_field(payload, "upstream_index")),
366 upstream_base_url: json_string_field(payload, "upstream_base_url"),
367 provider_id: json_string_field(payload, "provider_id"),
368 endpoint_id: json_string_field(payload, "endpoint_id"),
369 provider_endpoint_key: json_string_field(payload, "provider_endpoint_key"),
370 preference_group: json_u64_field(payload, "preference_group"),
371 model: json_string_field(payload, "model"),
372 }),
373 "retry_options" => Some(ControlTraceDetail::RetryOptions {
374 upstream_max_attempts: json_nested_u64_field(payload, &["upstream", "max_attempts"])
375 .and_then(|value| u32::try_from(value).ok()),
376 provider_max_attempts: json_nested_u64_field(payload, &["provider", "max_attempts"])
377 .and_then(|value| u32::try_from(value).ok()),
378 allow_cross_station_before_first_output: json_bool_field(
379 payload,
380 "allow_cross_station_before_first_output",
381 ),
382 }),
383 "lbs_for_request" => Some(ControlTraceDetail::LoadBalancerSelection {
384 mode: json_string_field(payload, "mode"),
385 pinned_source: json_string_field(payload, "pinned_source"),
386 pinned_name: json_string_field(payload, "pinned_name"),
387 selected_station: json_string_field(payload, "selected_station"),
388 selected_stations: json_string_vec_field(payload, "selected_stations"),
389 active_station: json_string_field(payload, "active_station"),
390 note: json_string_field(payload, "note"),
391 }),
392 "provider_runtime_override" => Some(ControlTraceDetail::ProviderRuntimeOverride {
393 provider_name: json_string_field(payload, "provider_name"),
394 endpoint_name: json_string_field(payload, "endpoint_name"),
395 base_urls: json_string_vec_field(payload, "base_urls"),
396 enabled: json_bool_field(payload, "enabled"),
397 clear_enabled: json_bool_field(payload, "clear_enabled").unwrap_or(false),
398 runtime_state: json_string_field(payload, "runtime_state"),
399 clear_runtime_state: json_bool_field(payload, "clear_runtime_state").unwrap_or(false),
400 }),
401 "route_executor_shadow_mismatch" => Some(route_executor_shadow_mismatch_detail(payload)),
402 "route_graph_selection_explain" => Some(route_graph_selection_explain_detail(payload)),
403 _ => Some(ControlTraceDetail::RetryEvent {
404 event_name,
405 station_name: json_string_field(payload, "station_name")
406 .or_else(|| json_string_field(payload, "selected_station")),
407 upstream_base_url: json_string_field(payload, "upstream_base_url"),
408 mode: json_string_field(payload, "mode"),
409 note: json_string_field(payload, "note"),
410 }),
411 }
412}
413
414fn route_graph_selection_explain_detail(payload: &JsonValue) -> ControlTraceDetail {
415 ControlTraceDetail::RouteGraphSelectionExplain {
416 request_model: json_string_field(payload, "request_model"),
417 continuity_class: json_nested_string_field(payload, &["continuity", "class"]),
418 provider_failover_allowed: payload
419 .get("continuity")
420 .and_then(|continuity| json_bool_field(continuity, "provider_failover_allowed")),
421 provider_failover_blocked_reason: json_nested_string_field(
422 payload,
423 &["continuity", "provider_failover_blocked_reason"],
424 ),
425 balance_signal_authoritative: payload
426 .get("continuity")
427 .and_then(|continuity| json_bool_field(continuity, "balance_signal_authoritative")),
428 affinity_policy: json_nested_string_field(payload, &["affinity", "policy"]),
429 affinity_source: json_nested_string_field(payload, &["affinity", "source"]),
430 affinity_provider_endpoint_key: json_nested_string_field(
431 payload,
432 &["affinity", "provider_endpoint_key"],
433 ),
434 selected_matches_affinity: payload
435 .get("affinity")
436 .and_then(|affinity| json_bool_field(affinity, "selected_matches_affinity")),
437 selected_provider_id: json_nested_string_field(payload, &["selected", "provider_id"]),
438 selected_endpoint_id: json_nested_string_field(payload, &["selected", "endpoint_id"]),
439 selected_provider_endpoint_key: json_nested_string_field(
440 payload,
441 &["selected", "provider_endpoint_key"],
442 ),
443 selected_preference_group: json_nested_u64_field(
444 payload,
445 &["selected", "preference_group"],
446 ),
447 skipped_higher_priority_groups: json_u64_vec_field(
448 payload,
449 "skipped_higher_priority_groups",
450 ),
451 skipped_higher_priority_candidates: json_array_field(
452 payload,
453 "skipped_higher_priority_candidates",
454 )
455 .iter()
456 .map(|candidate| ControlTraceRouteGraphSkippedCandidate {
457 provider_id: json_string_field(candidate, "provider_id"),
458 endpoint_id: json_string_field(candidate, "endpoint_id"),
459 provider_endpoint_key: json_string_field(candidate, "provider_endpoint_key"),
460 preference_group: json_u64_field(candidate, "preference_group"),
461 route_path: json_string_vec_field(candidate, "route_path"),
462 reasons: json_string_vec_field(candidate, "reasons"),
463 })
464 .collect(),
465 }
466}
467
468fn route_executor_shadow_mismatch_detail(payload: &JsonValue) -> ControlTraceDetail {
469 let legacy_attempts = json_array_field(payload, "legacy_attempts");
470 let executor_attempts = json_array_field(payload, "executor_attempts");
471 let first_mismatch_index = first_mismatch_index(legacy_attempts, executor_attempts);
472 let legacy_attempt = first_mismatch_index.and_then(|idx| legacy_attempts.get(idx));
473 let executor_attempt = first_mismatch_index.and_then(|idx| executor_attempts.get(idx));
474
475 ControlTraceDetail::RouteExecutorShadowMismatch {
476 request_model: json_string_field(payload, "request_model"),
477 legacy_attempt_count: legacy_attempts.len(),
478 executor_attempt_count: executor_attempts.len(),
479 first_mismatch_index,
480 legacy_station_name: legacy_attempt
481 .and_then(|attempt| json_string_field(attempt, "station_name")),
482 legacy_upstream_index: legacy_attempt
483 .and_then(|attempt| json_u64_field(attempt, "upstream_index")),
484 legacy_provider_id: legacy_attempt
485 .and_then(|attempt| json_string_field(attempt, "provider_id")),
486 executor_station_name: executor_attempt
487 .and_then(|attempt| json_string_field(attempt, "station_name")),
488 executor_upstream_index: executor_attempt
489 .and_then(|attempt| json_u64_field(attempt, "upstream_index")),
490 executor_provider_id: executor_attempt
491 .and_then(|attempt| json_string_field(attempt, "provider_id")),
492 }
493}
494
495fn first_mismatch_index(left: &[JsonValue], right: &[JsonValue]) -> Option<usize> {
496 let shared_len = left.len().min(right.len());
497 for idx in 0..shared_len {
498 if left[idx] != right[idx] {
499 return Some(idx);
500 }
501 }
502 (left.len() != right.len()).then_some(shared_len)
503}
504
505pub(super) fn append_control_trace_payload(
506 opt: RequestLogOptions,
507 kind: &'static str,
508 service: Option<&str>,
509 request_id: Option<u64>,
510 event: Option<&str>,
511 ts_ms: u64,
512 payload: JsonValue,
513) {
514 if !control_trace_enabled() {
515 return;
516 }
517 let path = control_trace_path();
518 ensure_log_parent(&path);
519 let entry = make_control_trace_entry(kind, service, request_id, event, ts_ms, payload);
520 if let Ok(line) = serde_json::to_string(&entry) {
521 let _ = append_json_line(&path, opt, &line);
522 }
523}
524
525fn make_control_trace_entry(
526 kind: &'static str,
527 service: Option<&str>,
528 request_id: Option<u64>,
529 event: Option<&str>,
530 ts_ms: u64,
531 payload: JsonValue,
532) -> ControlTraceLogEntry {
533 let detail = infer_control_trace_detail(kind, event, &payload);
534 let trace_id = derive_control_trace_id(None, service, request_id, &payload);
535 ControlTraceLogEntry {
536 ts_ms,
537 kind: kind.to_string(),
538 service: service.map(str::to_string),
539 request_id,
540 trace_id,
541 event: event.map(str::to_string),
542 detail,
543 payload,
544 }
545}
546
547pub fn log_retry_trace(mut event: JsonValue) {
548 let legacy_enabled = retry_trace_enabled();
549 let unified_enabled = control_trace_enabled();
550 if !legacy_enabled && !unified_enabled {
551 return;
552 }
553
554 if let JsonValue::Object(ref mut obj) = event {
555 obj.entry("ts_ms".to_string())
556 .or_insert_with(|| JsonValue::Number(serde_json::Number::from(now_ms())));
557 }
558
559 let ts_ms = json_u64_field(&event, "ts_ms").unwrap_or_else(now_ms);
560 let service = json_string_field(&event, "service");
561 let request_id = json_u64_field(&event, "request_id");
562 let event_name = json_string_field(&event, "event");
563 let opt = request_log_options();
564 let _guard = match log_lock().lock() {
565 Ok(g) => g,
566 Err(e) => e.into_inner(),
567 };
568
569 if legacy_enabled {
570 let path = retry_trace_path();
571 ensure_log_parent(&path);
572 if let Ok(line) = serde_json::to_string(&event) {
573 let _ = append_json_line(&path, opt, &line);
574 }
575 }
576
577 append_control_trace_payload(
578 opt,
579 "retry_trace",
580 service.as_deref(),
581 request_id,
582 event_name.as_deref(),
583 ts_ms,
584 event,
585 );
586}
587
588#[cfg(test)]
589mod tests {
590 use super::*;
591
592 #[test]
593 fn json_field_helpers_extract_string_and_numeric_values() {
594 let event = serde_json::json!({
595 "service": "codex",
596 "request_id": "42",
597 "ts_ms": 99,
598 });
599
600 assert_eq!(
601 json_string_field(&event, "service").as_deref(),
602 Some("codex")
603 );
604 assert_eq!(json_u64_field(&event, "request_id"), Some(42));
605 assert_eq!(json_u64_field(&event, "ts_ms"), Some(99));
606 }
607
608 #[test]
609 fn make_control_trace_entry_keeps_kind_event_and_request_id() {
610 let entry = make_control_trace_entry(
611 "retry_trace",
612 Some("codex"),
613 Some(7),
614 Some("attempt_select"),
615 123,
616 serde_json::json!({
617 "event": "attempt_select",
618 "service": "codex",
619 "request_id": 7,
620 "provider_id": "monthly",
621 "endpoint_id": "default",
622 "provider_endpoint_key": "codex/monthly/default",
623 "preference_group": 0,
624 "compatibility": {
625 "station_name": "routing",
626 "upstream_index": 0
627 },
628 }),
629 );
630
631 let value = serde_json::to_value(entry).expect("serialize control trace entry");
632 assert_eq!(value["kind"].as_str(), Some("retry_trace"));
633 assert_eq!(value["event"].as_str(), Some("attempt_select"));
634 assert_eq!(value["request_id"].as_u64(), Some(7));
635 assert_eq!(value["trace_id"].as_str(), Some("codex-7"));
636 assert_eq!(value["service"].as_str(), Some("codex"));
637 assert_eq!(value["payload"]["event"].as_str(), Some("attempt_select"));
638 assert_eq!(value["detail"]["type"].as_str(), Some("attempt_select"));
639 assert_eq!(
640 value["detail"]["provider_endpoint_key"].as_str(),
641 Some("codex/monthly/default")
642 );
643 assert_eq!(value["detail"]["preference_group"].as_u64(), Some(0));
644 assert_eq!(value["detail"]["station_name"].as_str(), Some("routing"));
645 assert_eq!(value["detail"]["upstream_index"].as_u64(), Some(0));
646 }
647
648 #[test]
649 fn hydrate_control_trace_entry_adds_trace_id_to_legacy_event() {
650 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
651 "ts_ms": 1,
652 "kind": "retry_trace",
653 "service": "codex",
654 "request_id": 7,
655 "event": "attempt_select",
656 "payload": {
657 "event": "attempt_select",
658 "station_name": "right"
659 }
660 }))
661 .expect("deserialize legacy control trace");
662
663 let entry = hydrate_control_trace_entry(entry);
664
665 assert_eq!(entry.trace_id.as_deref(), Some("codex-7"));
666 assert_eq!(
667 entry.detail,
668 Some(ControlTraceDetail::AttemptSelect {
669 station_name: Some("right".to_string()),
670 upstream_index: None,
671 upstream_base_url: None,
672 provider_id: None,
673 endpoint_id: None,
674 provider_endpoint_key: None,
675 preference_group: None,
676 model: None,
677 })
678 );
679 }
680
681 #[test]
682 fn resolved_trace_id_prefers_payload_trace_id() {
683 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
684 "ts_ms": 1,
685 "kind": "retry_trace",
686 "service": "codex",
687 "request_id": 7,
688 "payload": {
689 "trace_id": "external-123",
690 "service": "codex",
691 "request_id": 8
692 }
693 }))
694 .expect("deserialize control trace");
695
696 assert_eq!(entry.resolved_trace_id().as_deref(), Some("external-123"));
697 }
698
699 #[test]
700 fn control_trace_entry_resolved_detail_infers_request_completed_from_legacy_payload() {
701 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
702 "ts_ms": 1,
703 "kind": "request_completed",
704 "service": "codex",
705 "request_id": 11,
706 "event": "request_completed",
707 "payload": {
708 "method": "POST",
709 "path": "/v1/responses",
710 "status_code": 200,
711 "duration_ms": 321,
712 "station_name": "right",
713 "provider_id": "right",
714 "service_tier": {
715 "effective": "priority"
716 }
717 }
718 }))
719 .expect("deserialize control trace");
720
721 assert_eq!(
722 entry.resolved_detail(),
723 Some(ControlTraceDetail::RequestCompleted {
724 method: Some("POST".to_string()),
725 path: Some("/v1/responses".to_string()),
726 status_code: Some(200),
727 duration_ms: Some(321),
728 station_name: Some("right".to_string()),
729 provider_id: Some("right".to_string()),
730 upstream_base_url: None,
731 service_tier: ServiceTierLog {
732 requested: None,
733 effective: Some("priority".to_string()),
734 actual: None,
735 },
736 })
737 );
738 }
739
740 #[test]
741 fn control_trace_entry_resolved_detail_infers_provider_runtime_override() {
742 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
743 "ts_ms": 1,
744 "kind": "retry_trace",
745 "service": "codex",
746 "event": "provider_runtime_override",
747 "payload": {
748 "event": "provider_runtime_override",
749 "provider_name": "alpha",
750 "endpoint_name": "default",
751 "base_urls": ["https://alpha.example/v1"],
752 "enabled": false,
753 "clear_enabled": false,
754 "runtime_state": "breaker_open",
755 "clear_runtime_state": false
756 }
757 }))
758 .expect("deserialize provider runtime trace");
759
760 assert_eq!(
761 entry.resolved_detail(),
762 Some(ControlTraceDetail::ProviderRuntimeOverride {
763 provider_name: Some("alpha".to_string()),
764 endpoint_name: Some("default".to_string()),
765 base_urls: vec!["https://alpha.example/v1".to_string()],
766 enabled: Some(false),
767 clear_enabled: false,
768 runtime_state: Some("breaker_open".to_string()),
769 clear_runtime_state: false,
770 })
771 );
772 }
773
774 #[test]
775 fn control_trace_entry_resolved_detail_infers_route_graph_selection_explain() {
776 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
777 "ts_ms": 1,
778 "kind": "retry_trace",
779 "service": "codex",
780 "request_id": 42,
781 "event": "route_graph_selection_explain",
782 "payload": {
783 "event": "route_graph_selection_explain",
784 "service": "codex",
785 "request_id": 42,
786 "request_model": "gpt-5.4",
787 "continuity": {
788 "class": "stateless_or_session_preferred",
789 "provider_failover_allowed": true,
790 "provider_failover_blocked_reason": null,
791 "balance_signal_authoritative": false
792 },
793 "affinity": {
794 "policy": "preferred_group",
795 "provider_endpoint_key": "codex/chili/default",
796 "source": "session_route_affinity",
797 "selected_matches_affinity": false
798 },
799 "selected": {
800 "provider_id": "chili",
801 "endpoint_id": "default",
802 "provider_endpoint_key": "codex/chili/default",
803 "preference_group": 1,
804 "route_path": ["entry", "fallback"]
805 },
806 "skipped_higher_priority_groups": [0],
807 "skipped_higher_priority_candidates": [
808 {
809 "provider_id": "monthly",
810 "endpoint_id": "default",
811 "provider_endpoint_key": "codex/monthly/default",
812 "preference_group": 0,
813 "route_path": ["entry", "monthly"],
814 "reasons": ["usage_exhausted"]
815 }
816 ]
817 }
818 }))
819 .expect("deserialize route graph selection explain trace");
820
821 assert_eq!(
822 entry.resolved_detail(),
823 Some(ControlTraceDetail::RouteGraphSelectionExplain {
824 request_model: Some("gpt-5.4".to_string()),
825 continuity_class: Some("stateless_or_session_preferred".to_string()),
826 provider_failover_allowed: Some(true),
827 provider_failover_blocked_reason: None,
828 balance_signal_authoritative: Some(false),
829 affinity_policy: Some("preferred_group".to_string()),
830 affinity_source: Some("session_route_affinity".to_string()),
831 affinity_provider_endpoint_key: Some("codex/chili/default".to_string()),
832 selected_matches_affinity: Some(false),
833 selected_provider_id: Some("chili".to_string()),
834 selected_endpoint_id: Some("default".to_string()),
835 selected_provider_endpoint_key: Some("codex/chili/default".to_string()),
836 selected_preference_group: Some(1),
837 skipped_higher_priority_groups: vec![0],
838 skipped_higher_priority_candidates: vec![ControlTraceRouteGraphSkippedCandidate {
839 provider_id: Some("monthly".to_string()),
840 endpoint_id: Some("default".to_string()),
841 provider_endpoint_key: Some("codex/monthly/default".to_string()),
842 preference_group: Some(0),
843 route_path: vec!["entry".to_string(), "monthly".to_string()],
844 reasons: vec!["usage_exhausted".to_string()],
845 }],
846 })
847 );
848 }
849
850 #[test]
851 fn control_trace_entry_resolved_detail_infers_route_executor_shadow_mismatch() {
852 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
853 "ts_ms": 1,
854 "kind": "retry_trace",
855 "service": "codex",
856 "request_id": 19,
857 "event": "route_executor_shadow_mismatch",
858 "payload": {
859 "event": "route_executor_shadow_mismatch",
860 "service": "codex",
861 "request_id": 19,
862 "request_model": "gpt-5",
863 "legacy_attempts": [
864 {
865 "decision": "selected",
866 "station_name": "routing",
867 "upstream_index": 1,
868 "upstream_base_url": "https://legacy.example/v1",
869 "provider_id": "legacy"
870 }
871 ],
872 "executor_attempts": [
873 {
874 "decision": "selected",
875 "station_name": "routing",
876 "upstream_index": 0,
877 "upstream_base_url": "https://executor.example/v1",
878 "provider_id": "executor"
879 },
880 {
881 "decision": "selected",
882 "station_name": "routing",
883 "upstream_index": 1,
884 "upstream_base_url": "https://legacy.example/v1",
885 "provider_id": "legacy"
886 }
887 ]
888 }
889 }))
890 .expect("deserialize shadow mismatch trace");
891
892 let entry = hydrate_control_trace_entry(entry);
893 assert_eq!(
894 entry.resolved_detail(),
895 Some(ControlTraceDetail::RouteExecutorShadowMismatch {
896 request_model: Some("gpt-5".to_string()),
897 legacy_attempt_count: 1,
898 executor_attempt_count: 2,
899 first_mismatch_index: Some(0),
900 legacy_station_name: Some("routing".to_string()),
901 legacy_upstream_index: Some(1),
902 legacy_provider_id: Some("legacy".to_string()),
903 executor_station_name: Some("routing".to_string()),
904 executor_upstream_index: Some(0),
905 executor_provider_id: Some("executor".to_string()),
906 })
907 );
908
909 let value = serde_json::to_value(entry).expect("serialize shadow mismatch trace");
910 assert_eq!(
911 value["detail"]["type"].as_str(),
912 Some("route_executor_shadow_mismatch")
913 );
914 }
915
916 #[test]
917 fn first_mismatch_index_reports_length_mismatch_after_shared_prefix() {
918 let left = vec![serde_json::json!({"provider_id": "a"})];
919 let right = vec![
920 serde_json::json!({"provider_id": "a"}),
921 serde_json::json!({"provider_id": "b"}),
922 ];
923
924 assert_eq!(first_mismatch_index(&left, &right), Some(1));
925 assert_eq!(first_mismatch_index(&left, &left), None);
926 }
927}