1use super::*;
2
3#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
4#[serde(tag = "type", rename_all = "snake_case")]
5pub enum ControlTraceDetail {
6 RequestCompleted {
7 method: Option<String>,
8 path: Option<String>,
9 status_code: Option<u16>,
10 duration_ms: Option<u64>,
11 station_name: Option<String>,
12 provider_id: Option<String>,
13 upstream_base_url: Option<String>,
14 service_tier: ServiceTierLog,
15 },
16 RetryOptions {
17 upstream_max_attempts: Option<u32>,
18 provider_max_attempts: Option<u32>,
19 allow_cross_station_before_first_output: Option<bool>,
20 },
21 AttemptSelect {
22 station_name: Option<String>,
23 upstream_index: Option<u64>,
24 upstream_base_url: Option<String>,
25 provider_id: Option<String>,
26 endpoint_id: Option<String>,
27 provider_endpoint_key: Option<String>,
28 preference_group: Option<u64>,
29 model: Option<String>,
30 },
31 LoadBalancerSelection {
32 mode: Option<String>,
33 pinned_source: Option<String>,
34 pinned_name: Option<String>,
35 selected_station: Option<String>,
36 selected_stations: Vec<String>,
37 active_station: Option<String>,
38 note: Option<String>,
39 },
40 ProviderRuntimeOverride {
41 provider_name: Option<String>,
42 endpoint_name: Option<String>,
43 base_urls: Vec<String>,
44 enabled: Option<bool>,
45 clear_enabled: bool,
46 runtime_state: Option<String>,
47 clear_runtime_state: bool,
48 },
49 RouteExecutorShadowMismatch {
50 request_model: Option<String>,
51 legacy_attempt_count: usize,
52 executor_attempt_count: usize,
53 first_mismatch_index: Option<usize>,
54 legacy_station_name: Option<String>,
55 legacy_upstream_index: Option<u64>,
56 legacy_provider_id: Option<String>,
57 executor_station_name: Option<String>,
58 executor_upstream_index: Option<u64>,
59 executor_provider_id: Option<String>,
60 },
61 RouteGraphSelectionExplain {
62 request_model: Option<String>,
63 affinity_policy: Option<String>,
64 affinity_provider_endpoint_key: Option<String>,
65 selected_matches_affinity: Option<bool>,
66 selected_provider_id: Option<String>,
67 selected_endpoint_id: Option<String>,
68 selected_provider_endpoint_key: Option<String>,
69 selected_preference_group: Option<u64>,
70 skipped_higher_priority_groups: Vec<u64>,
71 skipped_higher_priority_candidates: Vec<ControlTraceRouteGraphSkippedCandidate>,
72 },
73 RetryEvent {
74 event_name: String,
75 station_name: Option<String>,
76 upstream_base_url: Option<String>,
77 mode: Option<String>,
78 note: Option<String>,
79 },
80}
81
82#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
83pub struct ControlTraceRouteGraphSkippedCandidate {
84 pub provider_id: Option<String>,
85 pub endpoint_id: Option<String>,
86 pub provider_endpoint_key: Option<String>,
87 pub preference_group: Option<u64>,
88 pub route_path: Vec<String>,
89 pub reasons: Vec<String>,
90}
91
92#[derive(Debug, Clone, Serialize, Deserialize)]
93pub struct ControlTraceLogEntry {
94 pub ts_ms: u64,
95 pub kind: String,
96 #[serde(skip_serializing_if = "Option::is_none")]
97 pub service: Option<String>,
98 #[serde(skip_serializing_if = "Option::is_none")]
99 pub request_id: Option<u64>,
100 #[serde(default, skip_serializing_if = "Option::is_none")]
101 pub trace_id: Option<String>,
102 #[serde(skip_serializing_if = "Option::is_none")]
103 pub event: Option<String>,
104 #[serde(default, skip_serializing_if = "Option::is_none")]
105 pub detail: Option<ControlTraceDetail>,
106 #[serde(default)]
107 pub payload: JsonValue,
108}
109
110impl ControlTraceLogEntry {
111 pub fn resolved_trace_id(&self) -> Option<String> {
112 derive_control_trace_id(
113 self.trace_id.as_deref(),
114 self.service.as_deref(),
115 self.request_id,
116 &self.payload,
117 )
118 }
119
120 pub fn resolved_detail(&self) -> Option<ControlTraceDetail> {
121 self.detail.clone().or_else(|| {
122 infer_control_trace_detail(self.kind.as_str(), self.event.as_deref(), &self.payload)
123 })
124 }
125}
126
127pub fn control_trace_path() -> PathBuf {
128 std::env::var("CODEX_HELPER_CONTROL_TRACE_PATH")
129 .ok()
130 .map(|s| s.trim().to_string())
131 .filter(|s| !s.is_empty())
132 .map(PathBuf::from)
133 .unwrap_or_else(|| proxy_home_dir().join("logs").join("control_trace.jsonl"))
134}
135
136fn control_trace_enabled() -> bool {
137 static ENABLED: OnceLock<bool> = OnceLock::new();
138 *ENABLED.get_or_init(|| env_bool_default("CODEX_HELPER_CONTROL_TRACE", true))
139}
140
141fn retry_trace_enabled() -> bool {
142 static ENABLED: OnceLock<bool> = OnceLock::new();
143 *ENABLED.get_or_init(|| env_bool("CODEX_HELPER_RETRY_TRACE"))
144}
145
146fn retry_trace_path() -> PathBuf {
147 std::env::var("CODEX_HELPER_RETRY_TRACE_PATH")
148 .ok()
149 .map(|s| s.trim().to_string())
150 .filter(|s| !s.is_empty())
151 .map(PathBuf::from)
152 .unwrap_or_else(|| proxy_home_dir().join("logs").join("retry_trace.jsonl"))
153}
154
155fn control_trace_read_window(limit: usize) -> usize {
156 limit.saturating_mul(4).clamp(80, 400)
157}
158
159pub fn read_recent_control_trace_entries(
160 limit: usize,
161) -> anyhow::Result<Vec<ControlTraceLogEntry>> {
162 use std::collections::VecDeque;
163 use std::io::{BufRead, BufReader};
164
165 let path = control_trace_path();
166 if !path.exists() {
167 return Ok(Vec::new());
168 }
169
170 let file = fs::File::open(&path)?;
171 let reader = BufReader::new(file);
172 let mut ring = VecDeque::with_capacity(control_trace_read_window(limit));
173 for line in reader.lines() {
174 let line = line?;
175 if line.trim().is_empty() {
176 continue;
177 }
178 if ring.len() == ring.capacity() {
179 ring.pop_front();
180 }
181 ring.push_back(line);
182 }
183
184 let mut out = Vec::new();
185 for line in ring {
186 let Ok(entry) = serde_json::from_str::<ControlTraceLogEntry>(&line) else {
187 continue;
188 };
189 out.push(hydrate_control_trace_entry(entry));
190 }
191 out.sort_by_key(|entry| std::cmp::Reverse(entry.ts_ms));
192 Ok(out)
193}
194
195fn hydrate_control_trace_entry(mut entry: ControlTraceLogEntry) -> ControlTraceLogEntry {
196 if entry.trace_id.is_none() {
197 entry.trace_id = entry.resolved_trace_id();
198 }
199 if entry.detail.is_none() {
200 entry.detail =
201 infer_control_trace_detail(entry.kind.as_str(), entry.event.as_deref(), &entry.payload);
202 }
203 entry
204}
205
206fn json_u64_field(value: &JsonValue, key: &str) -> Option<u64> {
207 value.get(key).and_then(|value| match value {
208 JsonValue::Number(number) => number.as_u64(),
209 JsonValue::String(text) => text.trim().parse::<u64>().ok(),
210 _ => None,
211 })
212}
213
214fn json_u16_field(value: &JsonValue, key: &str) -> Option<u16> {
215 json_u64_field(value, key).and_then(|value| u16::try_from(value).ok())
216}
217
218fn json_string_field(value: &JsonValue, key: &str) -> Option<String> {
219 value
220 .get(key)
221 .and_then(|value| value.as_str())
222 .map(str::trim)
223 .filter(|value| !value.is_empty())
224 .map(ToOwned::to_owned)
225}
226
227fn json_bool_field(value: &JsonValue, key: &str) -> Option<bool> {
228 value.get(key).and_then(|value| value.as_bool())
229}
230
231fn json_nested_string_field(value: &JsonValue, path: &[&str]) -> Option<String> {
232 let mut current = value;
233 for segment in path {
234 current = current.get(*segment)?;
235 }
236 current
237 .as_str()
238 .map(str::trim)
239 .filter(|value| !value.is_empty())
240 .map(ToOwned::to_owned)
241}
242
243fn json_nested_u64_field(value: &JsonValue, path: &[&str]) -> Option<u64> {
244 let mut current = value;
245 for segment in path {
246 current = current.get(*segment)?;
247 }
248 match current {
249 JsonValue::Number(number) => number.as_u64(),
250 JsonValue::String(text) => text.trim().parse::<u64>().ok(),
251 _ => None,
252 }
253}
254
255fn json_string_vec_field(value: &JsonValue, key: &str) -> Vec<String> {
256 value
257 .get(key)
258 .and_then(|value| value.as_array())
259 .map(|items| {
260 items
261 .iter()
262 .filter_map(|item| item.as_str())
263 .map(str::trim)
264 .filter(|item| !item.is_empty())
265 .map(ToOwned::to_owned)
266 .collect::<Vec<_>>()
267 })
268 .unwrap_or_default()
269}
270
271fn json_u64_vec_field(value: &JsonValue, key: &str) -> Vec<u64> {
272 value
273 .get(key)
274 .and_then(|value| value.as_array())
275 .map(|items| {
276 items
277 .iter()
278 .filter_map(|item| match item {
279 JsonValue::Number(number) => number.as_u64(),
280 JsonValue::String(text) => text.trim().parse::<u64>().ok(),
281 _ => None,
282 })
283 .collect::<Vec<_>>()
284 })
285 .unwrap_or_default()
286}
287
288fn json_array_field<'a>(value: &'a JsonValue, key: &str) -> &'a [JsonValue] {
289 value
290 .get(key)
291 .and_then(|value| value.as_array())
292 .map(Vec::as_slice)
293 .unwrap_or(&[])
294}
295
296fn derive_control_trace_id(
297 trace_id: Option<&str>,
298 service: Option<&str>,
299 request_id: Option<u64>,
300 payload: &JsonValue,
301) -> Option<String> {
302 trace_id
303 .map(str::trim)
304 .filter(|value| !value.is_empty())
305 .map(ToOwned::to_owned)
306 .or_else(|| json_string_field(payload, "trace_id"))
307 .or_else(|| {
308 let request_id = request_id.or_else(|| json_u64_field(payload, "request_id"))?;
309 let service = service
310 .map(str::to_string)
311 .or_else(|| json_string_field(payload, "service"))
312 .unwrap_or_default();
313 Some(request_trace_id(service.as_str(), request_id))
314 })
315}
316
317fn json_service_tier_field(value: &JsonValue) -> ServiceTierLog {
318 value
319 .get("service_tier")
320 .cloned()
321 .and_then(|value| serde_json::from_value::<ServiceTierLog>(value).ok())
322 .unwrap_or_default()
323}
324
325fn infer_control_trace_detail(
326 kind: &str,
327 event: Option<&str>,
328 payload: &JsonValue,
329) -> Option<ControlTraceDetail> {
330 match kind {
331 "request_completed" => Some(ControlTraceDetail::RequestCompleted {
332 method: json_string_field(payload, "method"),
333 path: json_string_field(payload, "path"),
334 status_code: json_u16_field(payload, "status_code"),
335 duration_ms: json_u64_field(payload, "duration_ms"),
336 station_name: json_string_field(payload, "station_name"),
337 provider_id: json_string_field(payload, "provider_id"),
338 upstream_base_url: json_string_field(payload, "upstream_base_url"),
339 service_tier: json_service_tier_field(payload),
340 }),
341 "retry_trace" => infer_retry_control_trace_detail(event, payload),
342 _ => None,
343 }
344}
345
346fn infer_retry_control_trace_detail(
347 event: Option<&str>,
348 payload: &JsonValue,
349) -> Option<ControlTraceDetail> {
350 let event_name = event
351 .map(str::to_string)
352 .or_else(|| json_string_field(payload, "event"))
353 .unwrap_or_else(|| "retry_trace".to_string());
354
355 match event_name.as_str() {
356 "attempt_select" => Some(ControlTraceDetail::AttemptSelect {
357 station_name: json_nested_string_field(payload, &["compatibility", "station_name"])
358 .or_else(|| json_string_field(payload, "station_name")),
359 upstream_index: json_nested_u64_field(payload, &["compatibility", "upstream_index"])
360 .or_else(|| json_u64_field(payload, "upstream_index")),
361 upstream_base_url: json_string_field(payload, "upstream_base_url"),
362 provider_id: json_string_field(payload, "provider_id"),
363 endpoint_id: json_string_field(payload, "endpoint_id"),
364 provider_endpoint_key: json_string_field(payload, "provider_endpoint_key"),
365 preference_group: json_u64_field(payload, "preference_group"),
366 model: json_string_field(payload, "model"),
367 }),
368 "retry_options" => Some(ControlTraceDetail::RetryOptions {
369 upstream_max_attempts: json_nested_u64_field(payload, &["upstream", "max_attempts"])
370 .and_then(|value| u32::try_from(value).ok()),
371 provider_max_attempts: json_nested_u64_field(payload, &["provider", "max_attempts"])
372 .and_then(|value| u32::try_from(value).ok()),
373 allow_cross_station_before_first_output: json_bool_field(
374 payload,
375 "allow_cross_station_before_first_output",
376 ),
377 }),
378 "lbs_for_request" => Some(ControlTraceDetail::LoadBalancerSelection {
379 mode: json_string_field(payload, "mode"),
380 pinned_source: json_string_field(payload, "pinned_source"),
381 pinned_name: json_string_field(payload, "pinned_name"),
382 selected_station: json_string_field(payload, "selected_station"),
383 selected_stations: json_string_vec_field(payload, "selected_stations"),
384 active_station: json_string_field(payload, "active_station"),
385 note: json_string_field(payload, "note"),
386 }),
387 "provider_runtime_override" => Some(ControlTraceDetail::ProviderRuntimeOverride {
388 provider_name: json_string_field(payload, "provider_name"),
389 endpoint_name: json_string_field(payload, "endpoint_name"),
390 base_urls: json_string_vec_field(payload, "base_urls"),
391 enabled: json_bool_field(payload, "enabled"),
392 clear_enabled: json_bool_field(payload, "clear_enabled").unwrap_or(false),
393 runtime_state: json_string_field(payload, "runtime_state"),
394 clear_runtime_state: json_bool_field(payload, "clear_runtime_state").unwrap_or(false),
395 }),
396 "route_executor_shadow_mismatch" => Some(route_executor_shadow_mismatch_detail(payload)),
397 "route_graph_selection_explain" => Some(route_graph_selection_explain_detail(payload)),
398 _ => Some(ControlTraceDetail::RetryEvent {
399 event_name,
400 station_name: json_string_field(payload, "station_name")
401 .or_else(|| json_string_field(payload, "selected_station")),
402 upstream_base_url: json_string_field(payload, "upstream_base_url"),
403 mode: json_string_field(payload, "mode"),
404 note: json_string_field(payload, "note"),
405 }),
406 }
407}
408
409fn route_graph_selection_explain_detail(payload: &JsonValue) -> ControlTraceDetail {
410 ControlTraceDetail::RouteGraphSelectionExplain {
411 request_model: json_string_field(payload, "request_model"),
412 affinity_policy: json_nested_string_field(payload, &["affinity", "policy"]),
413 affinity_provider_endpoint_key: json_nested_string_field(
414 payload,
415 &["affinity", "provider_endpoint_key"],
416 ),
417 selected_matches_affinity: payload
418 .get("affinity")
419 .and_then(|affinity| json_bool_field(affinity, "selected_matches_affinity")),
420 selected_provider_id: json_nested_string_field(payload, &["selected", "provider_id"]),
421 selected_endpoint_id: json_nested_string_field(payload, &["selected", "endpoint_id"]),
422 selected_provider_endpoint_key: json_nested_string_field(
423 payload,
424 &["selected", "provider_endpoint_key"],
425 ),
426 selected_preference_group: json_nested_u64_field(
427 payload,
428 &["selected", "preference_group"],
429 ),
430 skipped_higher_priority_groups: json_u64_vec_field(
431 payload,
432 "skipped_higher_priority_groups",
433 ),
434 skipped_higher_priority_candidates: json_array_field(
435 payload,
436 "skipped_higher_priority_candidates",
437 )
438 .iter()
439 .map(|candidate| ControlTraceRouteGraphSkippedCandidate {
440 provider_id: json_string_field(candidate, "provider_id"),
441 endpoint_id: json_string_field(candidate, "endpoint_id"),
442 provider_endpoint_key: json_string_field(candidate, "provider_endpoint_key"),
443 preference_group: json_u64_field(candidate, "preference_group"),
444 route_path: json_string_vec_field(candidate, "route_path"),
445 reasons: json_string_vec_field(candidate, "reasons"),
446 })
447 .collect(),
448 }
449}
450
451fn route_executor_shadow_mismatch_detail(payload: &JsonValue) -> ControlTraceDetail {
452 let legacy_attempts = json_array_field(payload, "legacy_attempts");
453 let executor_attempts = json_array_field(payload, "executor_attempts");
454 let first_mismatch_index = first_mismatch_index(legacy_attempts, executor_attempts);
455 let legacy_attempt = first_mismatch_index.and_then(|idx| legacy_attempts.get(idx));
456 let executor_attempt = first_mismatch_index.and_then(|idx| executor_attempts.get(idx));
457
458 ControlTraceDetail::RouteExecutorShadowMismatch {
459 request_model: json_string_field(payload, "request_model"),
460 legacy_attempt_count: legacy_attempts.len(),
461 executor_attempt_count: executor_attempts.len(),
462 first_mismatch_index,
463 legacy_station_name: legacy_attempt
464 .and_then(|attempt| json_string_field(attempt, "station_name")),
465 legacy_upstream_index: legacy_attempt
466 .and_then(|attempt| json_u64_field(attempt, "upstream_index")),
467 legacy_provider_id: legacy_attempt
468 .and_then(|attempt| json_string_field(attempt, "provider_id")),
469 executor_station_name: executor_attempt
470 .and_then(|attempt| json_string_field(attempt, "station_name")),
471 executor_upstream_index: executor_attempt
472 .and_then(|attempt| json_u64_field(attempt, "upstream_index")),
473 executor_provider_id: executor_attempt
474 .and_then(|attempt| json_string_field(attempt, "provider_id")),
475 }
476}
477
478fn first_mismatch_index(left: &[JsonValue], right: &[JsonValue]) -> Option<usize> {
479 let shared_len = left.len().min(right.len());
480 for idx in 0..shared_len {
481 if left[idx] != right[idx] {
482 return Some(idx);
483 }
484 }
485 (left.len() != right.len()).then_some(shared_len)
486}
487
488pub(super) fn append_control_trace_payload(
489 opt: RequestLogOptions,
490 kind: &'static str,
491 service: Option<&str>,
492 request_id: Option<u64>,
493 event: Option<&str>,
494 ts_ms: u64,
495 payload: JsonValue,
496) {
497 if !control_trace_enabled() {
498 return;
499 }
500 let path = control_trace_path();
501 ensure_log_parent(&path);
502 let entry = make_control_trace_entry(kind, service, request_id, event, ts_ms, payload);
503 if let Ok(line) = serde_json::to_string(&entry) {
504 let _ = append_json_line(&path, opt, &line);
505 }
506}
507
508fn make_control_trace_entry(
509 kind: &'static str,
510 service: Option<&str>,
511 request_id: Option<u64>,
512 event: Option<&str>,
513 ts_ms: u64,
514 payload: JsonValue,
515) -> ControlTraceLogEntry {
516 let detail = infer_control_trace_detail(kind, event, &payload);
517 let trace_id = derive_control_trace_id(None, service, request_id, &payload);
518 ControlTraceLogEntry {
519 ts_ms,
520 kind: kind.to_string(),
521 service: service.map(str::to_string),
522 request_id,
523 trace_id,
524 event: event.map(str::to_string),
525 detail,
526 payload,
527 }
528}
529
530pub fn log_retry_trace(mut event: JsonValue) {
531 let legacy_enabled = retry_trace_enabled();
532 let unified_enabled = control_trace_enabled();
533 if !legacy_enabled && !unified_enabled {
534 return;
535 }
536
537 if let JsonValue::Object(ref mut obj) = event {
538 obj.entry("ts_ms".to_string())
539 .or_insert_with(|| JsonValue::Number(serde_json::Number::from(now_ms())));
540 }
541
542 let ts_ms = json_u64_field(&event, "ts_ms").unwrap_or_else(now_ms);
543 let service = json_string_field(&event, "service");
544 let request_id = json_u64_field(&event, "request_id");
545 let event_name = json_string_field(&event, "event");
546 let opt = request_log_options();
547 let _guard = match log_lock().lock() {
548 Ok(g) => g,
549 Err(e) => e.into_inner(),
550 };
551
552 if legacy_enabled {
553 let path = retry_trace_path();
554 ensure_log_parent(&path);
555 if let Ok(line) = serde_json::to_string(&event) {
556 let _ = append_json_line(&path, opt, &line);
557 }
558 }
559
560 append_control_trace_payload(
561 opt,
562 "retry_trace",
563 service.as_deref(),
564 request_id,
565 event_name.as_deref(),
566 ts_ms,
567 event,
568 );
569}
570
571#[cfg(test)]
572mod tests {
573 use super::*;
574
575 #[test]
576 fn json_field_helpers_extract_string_and_numeric_values() {
577 let event = serde_json::json!({
578 "service": "codex",
579 "request_id": "42",
580 "ts_ms": 99,
581 });
582
583 assert_eq!(
584 json_string_field(&event, "service").as_deref(),
585 Some("codex")
586 );
587 assert_eq!(json_u64_field(&event, "request_id"), Some(42));
588 assert_eq!(json_u64_field(&event, "ts_ms"), Some(99));
589 }
590
591 #[test]
592 fn make_control_trace_entry_keeps_kind_event_and_request_id() {
593 let entry = make_control_trace_entry(
594 "retry_trace",
595 Some("codex"),
596 Some(7),
597 Some("attempt_select"),
598 123,
599 serde_json::json!({
600 "event": "attempt_select",
601 "service": "codex",
602 "request_id": 7,
603 "provider_id": "monthly",
604 "endpoint_id": "default",
605 "provider_endpoint_key": "codex/monthly/default",
606 "preference_group": 0,
607 "compatibility": {
608 "station_name": "routing",
609 "upstream_index": 0
610 },
611 }),
612 );
613
614 let value = serde_json::to_value(entry).expect("serialize control trace entry");
615 assert_eq!(value["kind"].as_str(), Some("retry_trace"));
616 assert_eq!(value["event"].as_str(), Some("attempt_select"));
617 assert_eq!(value["request_id"].as_u64(), Some(7));
618 assert_eq!(value["trace_id"].as_str(), Some("codex-7"));
619 assert_eq!(value["service"].as_str(), Some("codex"));
620 assert_eq!(value["payload"]["event"].as_str(), Some("attempt_select"));
621 assert_eq!(value["detail"]["type"].as_str(), Some("attempt_select"));
622 assert_eq!(
623 value["detail"]["provider_endpoint_key"].as_str(),
624 Some("codex/monthly/default")
625 );
626 assert_eq!(value["detail"]["preference_group"].as_u64(), Some(0));
627 assert_eq!(value["detail"]["station_name"].as_str(), Some("routing"));
628 assert_eq!(value["detail"]["upstream_index"].as_u64(), Some(0));
629 }
630
631 #[test]
632 fn hydrate_control_trace_entry_adds_trace_id_to_legacy_event() {
633 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
634 "ts_ms": 1,
635 "kind": "retry_trace",
636 "service": "codex",
637 "request_id": 7,
638 "event": "attempt_select",
639 "payload": {
640 "event": "attempt_select",
641 "station_name": "right"
642 }
643 }))
644 .expect("deserialize legacy control trace");
645
646 let entry = hydrate_control_trace_entry(entry);
647
648 assert_eq!(entry.trace_id.as_deref(), Some("codex-7"));
649 assert_eq!(
650 entry.detail,
651 Some(ControlTraceDetail::AttemptSelect {
652 station_name: Some("right".to_string()),
653 upstream_index: None,
654 upstream_base_url: None,
655 provider_id: None,
656 endpoint_id: None,
657 provider_endpoint_key: None,
658 preference_group: None,
659 model: None,
660 })
661 );
662 }
663
664 #[test]
665 fn resolved_trace_id_prefers_payload_trace_id() {
666 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
667 "ts_ms": 1,
668 "kind": "retry_trace",
669 "service": "codex",
670 "request_id": 7,
671 "payload": {
672 "trace_id": "external-123",
673 "service": "codex",
674 "request_id": 8
675 }
676 }))
677 .expect("deserialize control trace");
678
679 assert_eq!(entry.resolved_trace_id().as_deref(), Some("external-123"));
680 }
681
682 #[test]
683 fn control_trace_entry_resolved_detail_infers_request_completed_from_legacy_payload() {
684 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
685 "ts_ms": 1,
686 "kind": "request_completed",
687 "service": "codex",
688 "request_id": 11,
689 "event": "request_completed",
690 "payload": {
691 "method": "POST",
692 "path": "/v1/responses",
693 "status_code": 200,
694 "duration_ms": 321,
695 "station_name": "right",
696 "provider_id": "right",
697 "service_tier": {
698 "effective": "priority"
699 }
700 }
701 }))
702 .expect("deserialize control trace");
703
704 assert_eq!(
705 entry.resolved_detail(),
706 Some(ControlTraceDetail::RequestCompleted {
707 method: Some("POST".to_string()),
708 path: Some("/v1/responses".to_string()),
709 status_code: Some(200),
710 duration_ms: Some(321),
711 station_name: Some("right".to_string()),
712 provider_id: Some("right".to_string()),
713 upstream_base_url: None,
714 service_tier: ServiceTierLog {
715 requested: None,
716 effective: Some("priority".to_string()),
717 actual: None,
718 },
719 })
720 );
721 }
722
723 #[test]
724 fn control_trace_entry_resolved_detail_infers_provider_runtime_override() {
725 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
726 "ts_ms": 1,
727 "kind": "retry_trace",
728 "service": "codex",
729 "event": "provider_runtime_override",
730 "payload": {
731 "event": "provider_runtime_override",
732 "provider_name": "alpha",
733 "endpoint_name": "default",
734 "base_urls": ["https://alpha.example/v1"],
735 "enabled": false,
736 "clear_enabled": false,
737 "runtime_state": "breaker_open",
738 "clear_runtime_state": false
739 }
740 }))
741 .expect("deserialize provider runtime trace");
742
743 assert_eq!(
744 entry.resolved_detail(),
745 Some(ControlTraceDetail::ProviderRuntimeOverride {
746 provider_name: Some("alpha".to_string()),
747 endpoint_name: Some("default".to_string()),
748 base_urls: vec!["https://alpha.example/v1".to_string()],
749 enabled: Some(false),
750 clear_enabled: false,
751 runtime_state: Some("breaker_open".to_string()),
752 clear_runtime_state: false,
753 })
754 );
755 }
756
757 #[test]
758 fn control_trace_entry_resolved_detail_infers_route_graph_selection_explain() {
759 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
760 "ts_ms": 1,
761 "kind": "retry_trace",
762 "service": "codex",
763 "request_id": 42,
764 "event": "route_graph_selection_explain",
765 "payload": {
766 "event": "route_graph_selection_explain",
767 "service": "codex",
768 "request_id": 42,
769 "request_model": "gpt-5.4",
770 "affinity": {
771 "policy": "preferred_group",
772 "provider_endpoint_key": "codex/chili/default",
773 "selected_matches_affinity": false
774 },
775 "selected": {
776 "provider_id": "chili",
777 "endpoint_id": "default",
778 "provider_endpoint_key": "codex/chili/default",
779 "preference_group": 1,
780 "route_path": ["entry", "fallback"]
781 },
782 "skipped_higher_priority_groups": [0],
783 "skipped_higher_priority_candidates": [
784 {
785 "provider_id": "monthly",
786 "endpoint_id": "default",
787 "provider_endpoint_key": "codex/monthly/default",
788 "preference_group": 0,
789 "route_path": ["entry", "monthly"],
790 "reasons": ["usage_exhausted"]
791 }
792 ]
793 }
794 }))
795 .expect("deserialize route graph selection explain trace");
796
797 assert_eq!(
798 entry.resolved_detail(),
799 Some(ControlTraceDetail::RouteGraphSelectionExplain {
800 request_model: Some("gpt-5.4".to_string()),
801 affinity_policy: Some("preferred_group".to_string()),
802 affinity_provider_endpoint_key: Some("codex/chili/default".to_string()),
803 selected_matches_affinity: Some(false),
804 selected_provider_id: Some("chili".to_string()),
805 selected_endpoint_id: Some("default".to_string()),
806 selected_provider_endpoint_key: Some("codex/chili/default".to_string()),
807 selected_preference_group: Some(1),
808 skipped_higher_priority_groups: vec![0],
809 skipped_higher_priority_candidates: vec![ControlTraceRouteGraphSkippedCandidate {
810 provider_id: Some("monthly".to_string()),
811 endpoint_id: Some("default".to_string()),
812 provider_endpoint_key: Some("codex/monthly/default".to_string()),
813 preference_group: Some(0),
814 route_path: vec!["entry".to_string(), "monthly".to_string()],
815 reasons: vec!["usage_exhausted".to_string()],
816 }],
817 })
818 );
819 }
820
821 #[test]
822 fn control_trace_entry_resolved_detail_infers_route_executor_shadow_mismatch() {
823 let entry: ControlTraceLogEntry = serde_json::from_value(serde_json::json!({
824 "ts_ms": 1,
825 "kind": "retry_trace",
826 "service": "codex",
827 "request_id": 19,
828 "event": "route_executor_shadow_mismatch",
829 "payload": {
830 "event": "route_executor_shadow_mismatch",
831 "service": "codex",
832 "request_id": 19,
833 "request_model": "gpt-5",
834 "legacy_attempts": [
835 {
836 "decision": "selected",
837 "station_name": "routing",
838 "upstream_index": 1,
839 "upstream_base_url": "https://legacy.example/v1",
840 "provider_id": "legacy"
841 }
842 ],
843 "executor_attempts": [
844 {
845 "decision": "selected",
846 "station_name": "routing",
847 "upstream_index": 0,
848 "upstream_base_url": "https://executor.example/v1",
849 "provider_id": "executor"
850 },
851 {
852 "decision": "selected",
853 "station_name": "routing",
854 "upstream_index": 1,
855 "upstream_base_url": "https://legacy.example/v1",
856 "provider_id": "legacy"
857 }
858 ]
859 }
860 }))
861 .expect("deserialize shadow mismatch trace");
862
863 let entry = hydrate_control_trace_entry(entry);
864 assert_eq!(
865 entry.resolved_detail(),
866 Some(ControlTraceDetail::RouteExecutorShadowMismatch {
867 request_model: Some("gpt-5".to_string()),
868 legacy_attempt_count: 1,
869 executor_attempt_count: 2,
870 first_mismatch_index: Some(0),
871 legacy_station_name: Some("routing".to_string()),
872 legacy_upstream_index: Some(1),
873 legacy_provider_id: Some("legacy".to_string()),
874 executor_station_name: Some("routing".to_string()),
875 executor_upstream_index: Some(0),
876 executor_provider_id: Some("executor".to_string()),
877 })
878 );
879
880 let value = serde_json::to_value(entry).expect("serialize shadow mismatch trace");
881 assert_eq!(
882 value["detail"]["type"].as_str(),
883 Some("route_executor_shadow_mismatch")
884 );
885 }
886
887 #[test]
888 fn first_mismatch_index_reports_length_mismatch_after_shared_prefix() {
889 let left = vec![serde_json::json!({"provider_id": "a"})];
890 let right = vec![
891 serde_json::json!({"provider_id": "a"}),
892 serde_json::json!({"provider_id": "b"}),
893 ];
894
895 assert_eq!(first_mismatch_index(&left, &right), Some(1));
896 assert_eq!(first_mismatch_index(&left, &left), None);
897 }
898}