1use async_trait::async_trait;
16use std::sync::Arc;
17
18use crate::error::AiMuxError;
19use crate::generate::{GenerateTextOptions, GenerateTextResult, generate_text};
20use crate::language_model::LanguageModel;
21use crate::language_model_message::LanguageModelPrompt;
22use crate::message::{MessageContent, ModelMessage, ModelPrompt};
23use crate::options::{CallOptions, ToolChoice};
24use crate::recording::Recording;
25use crate::result::{GenerateContent, GenerateResult, StreamResult};
26use crate::stream_part::StreamPart;
27use crate::types::{FinishReason, FinishReasonUnified, ResponseMetadata, Usage};
28
29pub trait ReplayMatcher: Send + Sync {
33 fn r#match<'a>(
40 &self,
41 options: &CallOptions,
42 recordings: &'a [Recording],
43 ) -> Result<&'a Recording, AiMuxError>;
44}
45
46pub struct ExactMatcher {
56 provider: String,
57 model_id: String,
58}
59
60impl ExactMatcher {
61 pub fn new(provider: impl Into<String>, model_id: impl Into<String>) -> Self {
62 Self {
63 provider: provider.into(),
64 model_id: model_id.into(),
65 }
66 }
67}
68
69impl ReplayMatcher for ExactMatcher {
70 fn r#match<'a>(
71 &self,
72 options: &CallOptions,
73 recordings: &'a [Recording],
74 ) -> Result<&'a Recording, AiMuxError> {
75 let needle = canonical_call_key(options);
76 recordings
77 .iter()
78 .find(|r| {
79 r.provider.provider == self.provider
80 && r.provider.model_id == self.model_id
81 && canonical_keys_match(&canonical_recording_key(r), &needle)
82 })
83 .ok_or_else(|| {
84 AiMuxError::InvalidArgument("mock replay: no exact matching recording".into())
85 })
86 }
87}
88
89fn canonical_call_key(opts: &CallOptions) -> serde_json::Value {
97 serde_json::json!({
98 "prompt": serde_json::to_value(&opts.prompt).unwrap_or_default(),
99 "temperature": opts.temperature,
100 "max_output_tokens": opts.max_output_tokens,
101 "seed": opts.seed,
102 "response_format": serde_json::to_value(&opts.response_format).unwrap_or_default(),
103 "tools": serde_json::to_value(&opts.tools).unwrap_or_default(),
104 "tool_choice": serde_json::to_value(&opts.tool_choice).unwrap_or_default(),
105 "headers": serde_json::to_value(&opts.headers).unwrap_or_default(),
106 "provider_options": serde_json::to_value(&opts.provider_options).unwrap_or_default(),
107 "body_overrides": serde_json::to_value(&opts.body_overrides).unwrap_or_default(),
108 })
109}
110
111fn canonical_recording_key(rec: &Recording) -> serde_json::Value {
114 let o = &rec.input.options;
115 serde_json::json!({
116 "prompt": serde_json::to_value(&rec.input.prompt).unwrap_or_default(),
117 "temperature": o.get("temperature").cloned().unwrap_or_default(),
118 "max_output_tokens": o.get("max_output_tokens").cloned().unwrap_or_default(),
119 "seed": o.get("seed").cloned().unwrap_or_default(),
120 "response_format": o.get("response_format").cloned().unwrap_or_default(),
121 "tools": o.get("tools").cloned().unwrap_or_default(),
122 "tool_choice": o
123 .get("tool_choice")
124 .cloned()
125 .unwrap_or_else(|| serde_json::to_value(ToolChoice::default()).unwrap_or_default()),
126 "headers": o.get("headers").cloned().unwrap_or_default(),
127 "provider_options": o.get("provider_options").cloned().unwrap_or_default(),
128 "body_overrides": o.get("body_overrides").cloned().unwrap_or_default(),
129 })
130}
131
132fn redaction_aware_eq(rec: &serde_json::Value, req: &serde_json::Value) -> bool {
147 if rec.as_str() == Some("[REDACTED]") {
149 return !req.is_null();
150 }
151 if rec.is_null() || req.is_null() {
153 return rec.is_null() && req.is_null();
154 }
155 match (rec, req) {
157 (serde_json::Value::Object(a), serde_json::Value::Object(b)) => {
158 if a.len() != b.len() {
159 return false; }
161 a.iter().all(|(k, rv)| {
162 b.get(k)
163 .map(|qv| redaction_aware_eq(rv, qv))
164 .unwrap_or(false)
165 })
166 }
167 (serde_json::Value::Array(a), serde_json::Value::Array(b)) => {
168 a.len() == b.len() && a.iter().zip(b).all(|(x, y)| redaction_aware_eq(x, y))
169 }
170 (x, y) => x == y,
171 }
172}
173
174fn canonical_keys_match(rec_key: &serde_json::Value, call_key: &serde_json::Value) -> bool {
182 const REDACTED_FIELDS: [&str; 3] = ["headers", "provider_options", "body_overrides"];
183 let (ro, co) = match (rec_key.as_object(), call_key.as_object()) {
184 (Some(a), Some(b)) => (a, b),
185 _ => return rec_key == call_key,
186 };
187 if ro.len() != co.len() {
188 return false;
189 }
190 ro.iter().all(|(k, rv)| {
191 co.get(k)
192 .map(|qv| {
193 if REDACTED_FIELDS.contains(&k.as_str()) {
194 redaction_aware_eq(rv, qv)
195 } else {
196 rv == qv
197 }
198 })
199 .unwrap_or(false)
200 })
201}
202
203pub struct ScoreMatcher {
210 provider: String,
211 model_id: String,
212}
213
214impl ScoreMatcher {
215 pub fn new(provider: impl Into<String>, model_id: impl Into<String>) -> Self {
216 Self {
217 provider: provider.into(),
218 model_id: model_id.into(),
219 }
220 }
221}
222
223impl ReplayMatcher for ScoreMatcher {
224 fn r#match<'a>(
225 &self,
226 options: &CallOptions,
227 recordings: &'a [Recording],
228 ) -> Result<&'a Recording, AiMuxError> {
229 let mut best: Option<(&Recording, u64)> = None;
230 for r in recordings {
231 if r.provider.provider != self.provider || r.provider.model_id != self.model_id {
232 continue;
233 }
234 let score = match_score(options, r);
235 if score == 0 {
236 continue; }
238 if best.map(|(_, s)| score > s).unwrap_or(true) {
239 best = Some((r, score));
240 }
241 }
242 best.map(|(r, _)| r).ok_or_else(|| {
243 AiMuxError::InvalidArgument("mock replay: no scoring match for this input".into())
244 })
245 }
246}
247
248fn message_text(m: &crate::language_model_message::LanguageModelPromptMessage) -> String {
250 match m.content.first() {
251 Some(crate::content::ContentPart::Text { text, .. }) => text.clone(),
252 _ => serde_json::to_string(&m.content).unwrap_or_default(),
253 }
254}
255
256fn match_score(options: &CallOptions, rec: &Recording) -> u64 {
263 let common = options
265 .prompt
266 .iter()
267 .zip(rec.input.prompt.iter())
268 .take_while(|(a, b)| a.role == b.role && a.content == b.content)
269 .count();
270 let min_len = options.prompt.len().min(rec.input.prompt.len());
274 let lcp: u64 = if min_len == 0 {
275 0
276 } else {
277 let lcp_idx = common.min(min_len - 1);
278 match (options.prompt.get(lcp_idx), rec.input.prompt.get(lcp_idx)) {
279 (Some(a), Some(b)) => message_text(a)
280 .bytes()
281 .zip(message_text(b).bytes())
282 .take_while(|(x, y)| x == y)
283 .count() as u64,
284 _ => 0,
285 }
286 };
287 let prompt_relevance = (common as u64) * 100 + lcp;
288 if prompt_relevance == 0 {
289 return 0; }
291 let mut score = prompt_relevance;
293 if options.temperature
294 == rec
295 .input
296 .options
297 .get("temperature")
298 .and_then(serde_json::Value::as_f64)
299 {
300 score += 1;
301 }
302 score
303}
304
305pub struct PrefixMatcher {
311 provider: String,
312 model_id: String,
313}
314
315impl PrefixMatcher {
316 pub fn new(provider: impl Into<String>, model_id: impl Into<String>) -> Self {
317 Self {
318 provider: provider.into(),
319 model_id: model_id.into(),
320 }
321 }
322}
323
324impl ReplayMatcher for PrefixMatcher {
325 fn r#match<'a>(
326 &self,
327 options: &CallOptions,
328 recordings: &'a [Recording],
329 ) -> Result<&'a Recording, AiMuxError> {
330 let mut best: Option<(&Recording, usize)> = None;
331 for r in recordings {
332 if r.provider.provider != self.provider || r.provider.model_id != self.model_id {
333 continue;
334 }
335 let common = options
337 .prompt
338 .iter()
339 .zip(r.input.prompt.iter())
340 .take_while(|(a, b)| a.role == b.role && a.content == b.content)
341 .count();
342 if common < r.input.prompt.len() {
343 continue; }
345 if common > 0 && best.map(|(_, c)| common > c).unwrap_or(true) {
346 best = Some((r, common));
347 }
348 }
349 best.map(|(r, _)| r).ok_or_else(|| {
350 AiMuxError::InvalidArgument("mock replay: no prefix match for this input".into())
351 })
352 }
353}
354
355pub struct MockReplayModel {
359 provider: String,
360 model_id: String,
361 recordings: Vec<Recording>,
362 matcher: Arc<dyn ReplayMatcher>,
363}
364
365impl MockReplayModel {
366 pub fn new(
368 provider: impl Into<String>,
369 model_id: impl Into<String>,
370 recordings: Vec<Recording>,
371 ) -> Self {
372 let provider = provider.into();
373 let model_id = model_id.into();
374 let matcher = Arc::new(ScoreMatcher::new(provider.clone(), model_id.clone()));
375 Self {
376 provider,
377 model_id,
378 recordings,
379 matcher,
380 }
381 }
382
383 pub fn with_matcher(
385 provider: impl Into<String>,
386 model_id: impl Into<String>,
387 recordings: Vec<Recording>,
388 matcher: Arc<dyn ReplayMatcher>,
389 ) -> Self {
390 Self {
391 provider: provider.into(),
392 model_id: model_id.into(),
393 recordings,
394 matcher,
395 }
396 }
397
398 pub fn from_jsonl(path: &str) -> Result<Self, AiMuxError> {
406 let content = std::fs::read_to_string(path)
407 .map_err(|e| AiMuxError::InvalidArgument(format!("mock replay: {e}")))?;
408 let mut recordings = Vec::new();
409 for (idx, line) in content.lines().enumerate() {
410 if line.trim().is_empty() {
411 continue;
412 }
413 let rec: Recording = serde_json::from_str(line)
414 .map_err(|e| AiMuxError::JsonParse(format!("mock replay line {}: {e}", idx + 1)))?;
415 recordings.push(rec);
416 }
417 if recordings.is_empty() {
418 return Err(AiMuxError::InvalidArgument(
419 "mock replay: empty recording file".into(),
420 ));
421 }
422 let provider = recordings[0].provider.provider.clone();
423 let model_id = recordings[0].provider.model_id.clone();
424 Ok(Self::new(provider, model_id, recordings))
425 }
426
427 #[must_use]
429 pub fn recordings(&self) -> &[Recording] {
430 &self.recordings
431 }
432}
433
434#[async_trait]
435impl LanguageModel for MockReplayModel {
436 fn provider(&self) -> &str {
437 &self.provider
438 }
439
440 fn model_id(&self) -> &str {
441 &self.model_id
442 }
443
444 async fn do_generate(&self, options: &CallOptions) -> Result<GenerateResult, AiMuxError> {
445 let rec = self.matcher.r#match(options, &self.recordings)?;
446 rebuild_generate_result(rec)
447 }
448
449 async fn do_stream(&self, options: &CallOptions) -> Result<StreamResult, AiMuxError> {
450 let rec = self.matcher.r#match(options, &self.recordings)?;
451 rebuild_stream_result(rec)
452 }
453}
454
455fn last_response(rec: &Recording) -> Result<&crate::recording::ResponseRecord, AiMuxError> {
464 for e in rec.exchanges.iter().rev() {
465 if !e.finalized || e.error.is_some() {
466 continue;
467 }
468 if let Some(resp) = e.response.as_ref()
469 && (200..300).contains(&resp.status)
470 {
471 return Ok(resp);
472 }
473 }
474 Err(AiMuxError::InvalidArgument(
475 "mock replay: recording has no successful (2xx, finalized, error-free) response".into(),
476 ))
477}
478
479fn parse_finish(raw: Option<&str>) -> (FinishReasonUnified, Option<String>) {
481 let unified = match raw {
482 Some("stop") => FinishReasonUnified::Stop,
483 Some("length") => FinishReasonUnified::Length,
484 Some("content_filter") => FinishReasonUnified::ContentFilter,
485 Some("tool_calls") => FinishReasonUnified::ToolCalls,
486 Some("error") => FinishReasonUnified::Error,
487 _ => FinishReasonUnified::Other,
488 };
489 (unified, raw.map(std::string::ToString::to_string))
490}
491
492fn u32_from_json(v: &serde_json::Value, field: &str) -> Result<Option<u32>, AiMuxError> {
495 v.as_u64()
496 .map(|n| {
497 u32::try_from(n).map_err(|_| {
498 AiMuxError::JsonParse(format!("mock replay: usage '{field}' overflows u32: {n}"))
499 })
500 })
501 .transpose()
502}
503
504fn parse_usage(v: &serde_json::Value) -> Result<Usage, AiMuxError> {
506 let u = &v["usage"];
507 if u.is_null() {
508 return Ok(Usage::default());
509 }
510 let prompt_details = &u["prompt_tokens_details"];
511 let completion_details = &u["completion_tokens_details"];
512 Ok(Usage {
513 input_tokens: crate::types::TokenUsage {
514 total: u32_from_json(&u["prompt_tokens"], "prompt_tokens")?,
515 no_cache: None,
516 cache_read: u32_from_json(
517 &prompt_details["cached_tokens"],
518 "prompt_tokens_details.cached_tokens",
519 )?,
520 cache_write: None,
521 text: None,
522 reasoning: None,
523 },
524 output_tokens: crate::types::TokenUsage {
525 total: u32_from_json(&u["completion_tokens"], "completion_tokens")?,
526 no_cache: None,
527 cache_read: None,
528 cache_write: None,
529 text: None,
530 reasoning: u32_from_json(
531 &completion_details["reasoning_tokens"],
532 "completion_tokens_details.reasoning_tokens",
533 )?,
534 },
535 raw: Some(u.clone()),
536 })
537}
538
539fn rebuild_generate_result(rec: &Recording) -> Result<GenerateResult, AiMuxError> {
546 let resp = last_response(rec)?;
547 let body = resp
548 .body
549 .as_deref()
550 .ok_or_else(|| AiMuxError::InvalidArgument("mock replay: response has no body".into()))?;
551 let v: serde_json::Value = serde_json::from_str(body)
552 .map_err(|e| AiMuxError::JsonParse(format!("mock replay: {e}")))?;
553
554 let message = v
556 .get("choices")
557 .and_then(|c| c.get(0))
558 .and_then(|choice| choice.get("message"))
559 .ok_or_else(|| {
560 AiMuxError::UnsupportedFunctionality(
561 "mock replay: response is not an OpenAI chat.completions body (no choices[0].message)"
562 .into(),
563 )
564 })?;
565
566 let mut content = Vec::new();
569 if let Some(text) = message.get("content").and_then(|c| c.as_str())
570 && !text.is_empty()
571 {
572 content.push(GenerateContent::Text {
573 text: text.to_string(),
574 provider_metadata: None,
575 });
576 }
577 if let Some(tool_calls) = message.get("tool_calls").and_then(|t| t.as_array()) {
578 for tc in tool_calls {
579 let tool_call_id = tc
580 .get("id")
581 .and_then(|x| x.as_str())
582 .unwrap_or_default()
583 .to_string();
584 let function = tc.get("function").cloned().unwrap_or_default();
585 let tool_name = function
586 .get("name")
587 .and_then(|x| x.as_str())
588 .unwrap_or_default()
589 .to_string();
590 let args_str = function
591 .get("arguments")
592 .and_then(|x| x.as_str())
593 .unwrap_or("");
594 let input = args_str.to_string();
595 content.push(GenerateContent::ToolCall {
596 tool_call_id,
597 tool_name,
598 input,
599 provider_executed: None,
600 dynamic: None,
601 thought_signature: None,
602 provider_metadata: None,
603 });
604 }
605 }
606
607 let raw_finish = v["choices"][0]["finish_reason"]
608 .as_str()
609 .map(std::string::ToString::to_string);
610 let (unified, raw) = parse_finish(raw_finish.as_deref());
611
612 Ok(GenerateResult {
613 content,
614 finish_reason: FinishReason { unified, raw },
615 usage: parse_usage(&v)?,
616 warnings: Vec::new(),
617 provider_metadata: None,
618 response: ResponseMetadata {
619 id: v["id"].as_str().map(std::string::ToString::to_string),
620 timestamp: None,
621 model_id: v["model"].as_str().map(std::string::ToString::to_string),
622 },
623 request_body: None,
624 response_headers: None,
625 })
626}
627
628struct ToolCallAccumulator {
630 id: String,
631 name: String,
632 arguments: String,
633}
634
635fn rebuild_stream_result(rec: &Recording) -> Result<StreamResult, AiMuxError> {
648 let resp = last_response(rec)?;
649 let body = resp
650 .body
651 .as_deref()
652 .ok_or_else(|| AiMuxError::InvalidArgument("mock replay: response has no body".into()))?;
653
654 let mut parts: Vec<Result<StreamPart, AiMuxError>> = Vec::new();
655 parts.push(Ok(StreamPart::StreamStart { warnings: vec![] }));
656
657 let text_id = "0".to_string();
658 let reasoning_id = "reasoning-0".to_string();
659 let mut text_started = false;
660 let mut reasoning_started = false;
661 let mut response_meta_emitted = false;
662 let mut final_usage = Usage::default();
663 let mut final_finish: Option<FinishReason> = None;
664 let mut tool_calls: std::collections::HashMap<usize, ToolCallAccumulator> =
666 std::collections::HashMap::new();
667 let mut tool_order: Vec<usize> = Vec::new();
668 let mut saw_openai = false;
669 for block in body.split("\n\n") {
670 let line = block.trim();
671 let data = line.strip_prefix("data:").unwrap_or(line).trim();
672 if data.is_empty() || data == "[DONE]" {
673 continue;
674 }
675 let Ok(v) = serde_json::from_str::<serde_json::Value>(data) else {
676 continue;
677 };
678
679 let choices_arr = v.get("choices").and_then(|c| c.as_array());
681 if choices_arr.is_some() {
682 saw_openai = true;
683 }
684
685 if !response_meta_emitted
687 && (v.get("id").and_then(|x| x.as_str()).is_some()
688 || v.get("model").and_then(|x| x.as_str()).is_some())
689 {
690 response_meta_emitted = true;
691 parts.push(Ok(StreamPart::ResponseMetadata {
692 id: v["id"].as_str().map(std::string::ToString::to_string),
693 timestamp: None,
694 model_id: v["model"].as_str().map(std::string::ToString::to_string),
695 }));
696 }
697
698 if v.get("usage").is_some_and(|u| !u.is_null()) {
701 final_usage = parse_usage(&v)?;
702 }
703
704 if let Some(choices) = choices_arr {
705 for choice in choices {
706 let delta = &choice["delta"];
707
708 let reasoning_delta = delta
710 .get("reasoning_content")
711 .and_then(|x| x.as_str())
712 .or_else(|| delta.get("reasoning").and_then(|x| x.as_str()));
713 if let Some(reasoning) = reasoning_delta
714 && !reasoning.is_empty()
715 {
716 if !reasoning_started {
717 reasoning_started = true;
718 parts.push(Ok(StreamPart::ReasoningStart {
719 id: reasoning_id.clone(),
720 provider_metadata: None,
721 }));
722 }
723 parts.push(Ok(StreamPart::ReasoningDelta {
724 id: reasoning_id.clone(),
725 delta: reasoning.to_string(),
726 provider_metadata: None,
727 }));
728 }
729
730 if let Some(content) = delta.get("content").and_then(|x| x.as_str())
732 && !content.is_empty()
733 {
734 if reasoning_started {
735 parts.push(Ok(StreamPart::ReasoningEnd {
736 id: reasoning_id.clone(),
737 provider_metadata: None,
738 }));
739 reasoning_started = false;
740 }
741 if !text_started {
742 text_started = true;
743 parts.push(Ok(StreamPart::TextStart {
744 id: text_id.clone(),
745 provider_metadata: None,
746 }));
747 }
748 parts.push(Ok(StreamPart::TextDelta {
749 id: text_id.clone(),
750 delta: content.to_string(),
751 provider_metadata: None,
752 }));
753 }
754
755 if let Some(tcs) = delta.get("tool_calls").and_then(|t| t.as_array()) {
757 if reasoning_started {
758 parts.push(Ok(StreamPart::ReasoningEnd {
759 id: reasoning_id.clone(),
760 provider_metadata: None,
761 }));
762 reasoning_started = false;
763 }
764 for dtc in tcs {
765 let idx = dtc
766 .get("index")
767 .and_then(serde_json::Value::as_u64)
768 .map(|n| n as usize)
769 .unwrap_or(0);
770 let func = dtc.get("function").cloned().unwrap_or_default();
771 let is_new = !tool_calls.contains_key(&idx);
772 if is_new {
773 let id = dtc
774 .get("id")
775 .and_then(|x| x.as_str())
776 .unwrap_or_default()
777 .to_string();
778 let name = func
779 .get("name")
780 .and_then(|x| x.as_str())
781 .unwrap_or_default()
782 .to_string();
783 tool_calls.insert(
784 idx,
785 ToolCallAccumulator {
786 id: id.clone(),
787 name: name.clone(),
788 arguments: String::new(),
789 },
790 );
791 tool_order.push(idx);
792 parts.push(Ok(StreamPart::ToolInputStart {
793 id,
794 tool_name: name,
795 provider_executed: None,
796 dynamic: None,
797 title: None,
798 provider_metadata: None,
799 }));
800 }
801 if let Some(args) = func.get("arguments").and_then(|x| x.as_str())
804 && (!is_new || !args.is_empty())
805 && let Some(acc) = tool_calls.get_mut(&idx)
806 {
807 acc.arguments.push_str(args);
808 parts.push(Ok(StreamPart::ToolInputDelta {
809 id: acc.id.clone(),
810 delta: args.to_string(),
811 provider_metadata: None,
812 }));
813 }
814 }
815 }
816
817 if let Some(fr) = choice.get("finish_reason").and_then(|x| x.as_str()) {
820 if reasoning_started {
821 parts.push(Ok(StreamPart::ReasoningEnd {
822 id: reasoning_id.clone(),
823 provider_metadata: None,
824 }));
825 reasoning_started = false;
826 }
827 if text_started {
828 parts.push(Ok(StreamPart::TextEnd {
829 id: text_id.clone(),
830 provider_metadata: None,
831 }));
832 text_started = false;
833 }
834 let (unified, raw) = parse_finish(Some(fr));
835 final_finish = Some(FinishReason { unified, raw });
836 }
837 }
838 }
839 }
840
841 if reasoning_started {
843 parts.push(Ok(StreamPart::ReasoningEnd {
844 id: reasoning_id,
845 provider_metadata: None,
846 }));
847 }
848 if text_started {
849 parts.push(Ok(StreamPart::TextEnd {
850 id: text_id,
851 provider_metadata: None,
852 }));
853 }
854 for &i in &tool_order {
855 if let Some(acc) = tool_calls.get(&i) {
856 parts.push(Ok(StreamPart::ToolInputEnd {
857 id: acc.id.clone(),
858 provider_metadata: None,
859 }));
860 parts.push(Ok(StreamPart::ToolCall {
861 tool_call_id: acc.id.clone(),
862 tool_name: acc.name.clone(),
863 input: serde_json::Value::String(acc.arguments.clone()),
864 provider_executed: None,
865 dynamic: None,
866 thought_signature: None,
867 invalid: None,
868 error: None,
869 provider_metadata: None,
870 }));
871 }
872 }
873
874 if !saw_openai {
876 return Err(AiMuxError::UnsupportedFunctionality(
877 "mock replay: response is not an OpenAI chat.completions SSE stream".into(),
878 ));
879 }
880
881 parts.push(Ok(StreamPart::Finish {
882 finish_reason: final_finish.unwrap_or(FinishReason {
883 unified: FinishReasonUnified::Stop,
884 raw: None,
885 }),
886 usage: final_usage,
887 provider_metadata: None,
888 }));
889
890 Ok(StreamResult {
891 stream: Box::pin(futures::stream::iter(parts)),
892 request_body: None,
893 response_headers: None,
894 })
895}
896
897#[derive(Debug, Clone, Default)]
904pub struct ReplayOverrides {
905 pub prompt: Option<LanguageModelPrompt>,
907 pub temperature: Option<f64>,
909 pub max_output_tokens: Option<u32>,
911}
912
913pub async fn replay_with_model(
933 recording: &Recording,
934 model: &dyn LanguageModel,
935 overrides: Option<&ReplayOverrides>,
936) -> Result<GenerateTextResult, AiMuxError> {
937 let mut call_options: CallOptions = serde_json::from_value(recording.input.options.clone())
939 .map_err(|e| AiMuxError::JsonParse(format!("mock replay: input options invalid: {e}")))?;
940
941 if let Some(o) = overrides {
943 if let Some(prompt) = &o.prompt {
944 call_options.prompt = prompt.clone();
945 }
946 if o.temperature.is_some() {
947 call_options.temperature = o.temperature;
948 }
949 if o.max_output_tokens.is_some() {
950 call_options.max_output_tokens = o.max_output_tokens;
951 }
952 }
953
954 let prompt = model_prompt_from_lm(&call_options.prompt);
956 let options = generate_options_from_call_options(call_options);
957 generate_text(model, prompt, options).await
958}
959
960fn model_prompt_from_lm(prompt: &LanguageModelPrompt) -> ModelPrompt {
965 ModelPrompt::Messages(
966 prompt
967 .iter()
968 .map(|m| ModelMessage {
969 role: m.role,
970 content: MessageContent::Parts(m.content.clone()),
971 })
972 .collect(),
973 )
974}
975
976fn generate_options_from_call_options(o: CallOptions) -> GenerateTextOptions {
981 GenerateTextOptions {
982 max_output_tokens: o.max_output_tokens,
983 temperature: o.temperature,
984 stop_sequences: o.stop_sequences,
985 top_p: o.top_p,
986 top_k: o.top_k,
987 presence_penalty: o.presence_penalty,
988 frequency_penalty: o.frequency_penalty,
989 response_format: o.response_format,
990 seed: o.seed,
991 tools: o.tools,
992 tool_choice: if o.tool_choice == crate::tool::ToolChoice::Auto {
993 None
994 } else {
995 Some(o.tool_choice)
996 },
997 headers: o.headers,
998 provider_options: o.provider_options,
999 reasoning: o.reasoning,
1000 instructions: None,
1001 body_overrides: o.body_overrides,
1002 max_retries: o.max_retries,
1003 timeout: o.timeout,
1004 session_id: o.session_id,
1005 abort_signal: None,
1006 include_raw_chunks: o.include_raw_chunks,
1007 repair_tool_call: None,
1008 }
1009}
1010
1011#[cfg(test)]
1014mod tests {
1015 use super::*;
1016 use crate::generate::GenerateTextOptions;
1017 use crate::recording::{
1018 HttpExchange, HttpRecord, InputRecord, ProviderRecord, ResponseRecord, TimingRecord,
1019 };
1020 use futures::StreamExt;
1021
1022 fn sample_options(text: &str, temperature: Option<f64>) -> CallOptions {
1023 let prompt = vec![crate::language_model_message::LanguageModelPromptMessage {
1024 role: crate::message::Role::User,
1025 content: vec![crate::content::ContentPart::Text {
1026 text: text.to_string(),
1027 provider_options: None,
1028 }],
1029 provider_options: None,
1030 }];
1031 GenerateTextOptions {
1032 temperature,
1033 ..Default::default()
1034 }
1035 .into_call_options(prompt)
1036 }
1037
1038 fn openai_recording(trace_id: &str, prompt_text: &str, reply: &str, finish: &str) -> Recording {
1040 let input_prompt: crate::language_model_message::LanguageModelPrompt =
1041 vec![crate::language_model_message::LanguageModelPromptMessage {
1042 role: crate::message::Role::User,
1043 content: vec![crate::content::ContentPart::Text {
1044 text: prompt_text.to_string(),
1045 provider_options: None,
1046 }],
1047 provider_options: None,
1048 }];
1049 let body = serde_json::json!({
1050 "id": "chatcmpl-mock",
1051 "object": "chat.completion",
1052 "model": "gpt-4o",
1053 "choices": [{
1054 "index": 0,
1055 "message": { "role": "assistant", "content": reply },
1056 "finish_reason": finish
1057 }],
1058 "usage": { "prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8 }
1059 });
1060 Recording {
1061 schema: crate::recording::RECORDING_SCHEMA,
1062 call_id: trace_id.to_string(),
1063 recorded_at: "2026-08-06T00:00:00Z".to_string(),
1064 input: InputRecord {
1065 prompt: input_prompt,
1066 options: serde_json::json!({ "temperature": 0.7 }),
1067 },
1068 provider: ProviderRecord {
1069 provider: "openai".into(),
1070 model_id: "gpt-4o".into(),
1071 base_url: None,
1072 api_key_source: "none".into(),
1073 profile: None,
1074 provider_options: None,
1075 },
1076 exchanges: vec![HttpExchange {
1077 step: None,
1078 attempt: 0,
1079 exchange_index: 0,
1080 request: HttpRecord {
1081 method: "post".into(),
1082 url: "https://api.openai.com/v1/chat/completions".into(),
1083 headers: vec![],
1084 body: Some("{}".into()),
1085 },
1086 response: Some(ResponseRecord {
1087 status: 200,
1088 headers: vec![],
1089 body: Some(body.to_string()),
1090 stream_chunks: None,
1091 ttfb_ms: None,
1092 }),
1093 timing: TimingRecord {
1094 latency_ms: 10,
1095 ttfb_ms: None,
1096 },
1097 error: None,
1098 finalized: true,
1099 }],
1100 outcome: crate::recording::OutcomeRecord {
1101 status: crate::recording::OutcomeStatus::Success,
1102 finish_reason: Some("stop".into()),
1103 error: None,
1104 error_value: None,
1105 usage: None,
1106 },
1107 complete: true,
1108 transport_closed: true,
1109 session_id: None,
1110 step: None,
1111 }
1112 }
1113
1114 #[test]
1115 fn exact_matcher_hits_only_identical_prompt() {
1116 let recs = [openai_recording("t1", "ping", "pong", "stop")];
1119 let matcher = ExactMatcher::new("openai", "gpt-4o");
1120 assert!(
1121 matcher
1122 .r#match(&sample_options("ping", Some(0.7)), &recs)
1123 .is_ok()
1124 );
1125 assert!(
1127 matcher
1128 .r#match(&sample_options("pong", Some(0.7)), &recs)
1129 .is_err()
1130 );
1131 }
1132
1133 #[test]
1134 fn exact_matcher_different_temperature_misses() {
1135 let recs = [openai_recording("t1", "ping", "pong", "stop")]; let matcher = ExactMatcher::new("openai", "gpt-4o");
1138 assert!(
1139 matcher
1140 .r#match(&sample_options("ping", Some(0.1)), &recs)
1141 .is_err()
1142 );
1143 assert!(
1145 matcher
1146 .r#match(&sample_options("ping", Some(0.7)), &recs)
1147 .is_ok()
1148 );
1149 }
1150
1151 #[test]
1152 fn exact_matcher_different_options_misses() {
1153 let mut rec = openai_recording("t1", "ping", "pong", "stop");
1155 let mut call = sample_options("ping", Some(0.7));
1158 call.max_output_tokens = Some(128);
1159 rec.input.options = serde_json::to_value(&call).unwrap();
1160 let recs = [rec];
1161 let matcher = ExactMatcher::new("openai", "gpt-4o");
1162
1163 let mut req = sample_options("ping", Some(0.7));
1165 req.max_output_tokens = Some(256);
1166 assert!(matcher.r#match(&req, &recs).is_err());
1167
1168 let mut req2 = sample_options("ping", Some(0.7));
1170 req2.max_output_tokens = Some(128);
1171 assert!(matcher.r#match(&req2, &recs).is_ok());
1172 }
1173
1174 #[test]
1175 fn exact_matcher_different_headers_misses() {
1176 let mut rec = openai_recording("t1", "ping", "pong", "stop");
1178 let mut call = sample_options("ping", Some(0.7));
1179 call.headers = Some([("x-custom".into(), "a".into())].into_iter().collect());
1180 rec.input.options = serde_json::to_value(&call).unwrap();
1181 let recs = [rec];
1182 let matcher = ExactMatcher::new("openai", "gpt-4o");
1183
1184 let mut req = sample_options("ping", Some(0.7));
1186 req.headers = Some([("x-custom".into(), "b".into())].into_iter().collect());
1187 assert!(matcher.r#match(&req, &recs).is_err());
1188
1189 let mut req2 = sample_options("ping", Some(0.7));
1191 req2.headers = Some([("x-custom".into(), "a".into())].into_iter().collect());
1192 assert!(matcher.r#match(&req2, &recs).is_ok());
1193
1194 let mut req3 = sample_options("ping", Some(0.7));
1196 req3.headers = Some(
1197 [
1198 ("x-custom".into(), "a".into()),
1199 ("x-extra".into(), "z".into()),
1200 ]
1201 .into_iter()
1202 .collect(),
1203 );
1204 assert!(matcher.r#match(&req3, &recs).is_err());
1205 }
1206
1207 #[test]
1208 fn exact_matcher_different_provider_options_misses() {
1209 let mut rec = openai_recording("t1", "ping", "pong", "stop");
1211 let mut call = sample_options("ping", Some(0.7));
1212 call.provider_options = Some(
1213 [("openai".into(), serde_json::json!({ "foo": 1 }))]
1214 .into_iter()
1215 .collect(),
1216 );
1217 rec.input.options = serde_json::to_value(&call).unwrap();
1218 let recs = [rec];
1219 let matcher = ExactMatcher::new("openai", "gpt-4o");
1220
1221 let mut req = sample_options("ping", Some(0.7));
1223 req.provider_options = Some(
1224 [("openai".into(), serde_json::json!({ "foo": 2 }))]
1225 .into_iter()
1226 .collect(),
1227 );
1228 assert!(matcher.r#match(&req, &recs).is_err());
1229
1230 let mut req2 = sample_options("ping", Some(0.7));
1232 req2.provider_options = Some(
1233 [("openai".into(), serde_json::json!({ "foo": 1 }))]
1234 .into_iter()
1235 .collect(),
1236 );
1237 assert!(matcher.r#match(&req2, &recs).is_ok());
1238 }
1239
1240 #[test]
1241 fn exact_matcher_redacted_values_match_any() {
1242 let mut rec = openai_recording("t1", "ping", "pong", "stop");
1245 let mut call = sample_options("ping", Some(0.7));
1246 call.headers = Some(
1247 [
1248 ("authorization".into(), "sk-secret".into()),
1249 ("content-type".into(), "application/json".into()),
1250 ]
1251 .into_iter()
1252 .collect(),
1253 );
1254 let mut opts_val = serde_json::to_value(&call).unwrap();
1258 if let Some(h) = opts_val.get_mut("headers") {
1259 *h = crate::recording::redact_json(h.clone());
1260 }
1261 rec.input.options = opts_val;
1262 assert_eq!(
1264 rec.input.options["headers"]["authorization"],
1265 serde_json::json!("[REDACTED]")
1266 );
1267 let recs = [rec];
1268 let matcher = ExactMatcher::new("openai", "gpt-4o");
1269
1270 let mut req = sample_options("ping", Some(0.7));
1272 req.headers = Some(
1273 [
1274 ("authorization".into(), "sk-other".into()),
1275 ("content-type".into(), "application/json".into()),
1276 ]
1277 .into_iter()
1278 .collect(),
1279 );
1280 assert!(matcher.r#match(&req, &recs).is_ok());
1281
1282 let mut req2 = sample_options("ping", Some(0.7));
1284 req2.headers = Some(
1285 [
1286 ("authorization".into(), "sk-other".into()),
1287 ("content-type".into(), "text/plain".into()),
1288 ]
1289 .into_iter()
1290 .collect(),
1291 );
1292 assert!(matcher.r#match(&req2, &recs).is_err());
1293
1294 let mut req3 = sample_options("ping", Some(0.7));
1296 req3.headers = Some(
1297 [("content-type".into(), "application/json".into())]
1298 .into_iter()
1299 .collect(),
1300 );
1301 assert!(matcher.r#match(&req3, &recs).is_err());
1302
1303 let req4 = sample_options("ping", Some(0.7));
1305 assert!(matcher.r#match(&req4, &recs).is_err());
1306 }
1307
1308 #[test]
1309 fn score_matcher_prefers_longer_prefix_and_ties_to_first() {
1310 let recs = [
1311 openai_recording("t1", "hello", "a", "stop"),
1312 openai_recording("t2", "hello world", "b", "stop"),
1313 ];
1314 let matcher = ScoreMatcher::new("openai", "gpt-4o");
1315 let hit = matcher
1317 .r#match(&sample_options("hello world!", None), &recs)
1318 .unwrap();
1319 assert_eq!(hit.call_id, "t2");
1320 let hit = matcher
1322 .r#match(&sample_options("hello", None), &recs)
1323 .unwrap();
1324 assert_eq!(hit.call_id, "t1");
1325 let hit = matcher
1327 .r#match(&sample_options("hello", Some(0.7)), &recs)
1328 .unwrap();
1329 assert_eq!(hit.call_id, "t1");
1330 }
1331
1332 #[test]
1333 fn match_score_empty_prompt_no_underflow() {
1334 let rec = openai_recording("t1", "hello", "a", "stop");
1336 let empty_req = GenerateTextOptions::default().into_call_options(vec![]);
1337 assert_eq!(match_score(&empty_req, &rec), 0);
1339 let mut rec_empty = rec.clone();
1341 rec_empty.input.prompt = vec![];
1342 assert_eq!(match_score(&sample_options("hello", None), &rec_empty), 0);
1343 assert_eq!(match_score(&empty_req, &rec_empty), 0);
1345 }
1346
1347 #[test]
1348 fn score_matcher_unrelated_prompt_with_matching_temperature_misses() {
1349 let mut rec = openai_recording("t1", "alpha", "a", "stop");
1352 rec.input.options = serde_json::json!({}); let recs = [rec];
1354 let matcher = ScoreMatcher::new("openai", "gpt-4o");
1355 assert!(
1357 matcher
1358 .r#match(&sample_options("zzzzz", None), &recs)
1359 .is_err()
1360 );
1361 }
1362
1363 #[test]
1364 fn score_matcher_related_prompt_with_matching_temperature_hits() {
1365 let recs = [openai_recording("t1", "hello", "a", "stop")]; let matcher = ScoreMatcher::new("openai", "gpt-4o");
1368 let hit = matcher
1369 .r#match(&sample_options("hello", Some(0.7)), &recs)
1370 .unwrap();
1371 assert_eq!(hit.call_id, "t1");
1372 }
1373
1374 #[test]
1375 fn prefix_matcher_matches_message_prefix() {
1376 let rec = openai_recording("t1", "hello", "a", "stop"); let matcher = PrefixMatcher::new("openai", "gpt-4o");
1378 let prompt = vec![
1380 crate::language_model_message::LanguageModelPromptMessage {
1381 role: crate::message::Role::User,
1382 content: vec![crate::content::ContentPart::Text {
1383 text: "hello".into(),
1384 provider_options: None,
1385 }],
1386 provider_options: None,
1387 },
1388 crate::language_model_message::LanguageModelPromptMessage {
1389 role: crate::message::Role::User,
1390 content: vec![crate::content::ContentPart::Text {
1391 text: "world".into(),
1392 provider_options: None,
1393 }],
1394 provider_options: None,
1395 },
1396 ];
1397 let opts = GenerateTextOptions::default().into_call_options(prompt);
1398 let recs = [rec.clone()];
1399 let hit = matcher.r#match(&opts, &recs).unwrap();
1400 assert_eq!(hit.call_id, "t1");
1401
1402 let opts2 = GenerateTextOptions::default().into_call_options(vec![
1404 crate::language_model_message::LanguageModelPromptMessage {
1405 role: crate::message::Role::User,
1406 content: vec![crate::content::ContentPart::Text {
1407 text: "hi".into(),
1408 provider_options: None,
1409 }],
1410 provider_options: None,
1411 },
1412 ]);
1413 assert!(matcher.r#match(&opts2, &[rec]).is_err());
1414 }
1415
1416 #[test]
1417 fn prefix_matcher_prefers_longest_prefix() {
1418 let mut rec_a = openai_recording("ta", "hello", "a", "stop");
1420 let mut rec_b = openai_recording("tb", "hello world", "b", "stop");
1421 let mk = |text: &str| crate::language_model_message::LanguageModelPromptMessage {
1422 role: crate::message::Role::User,
1423 content: vec![crate::content::ContentPart::Text {
1424 text: text.into(),
1425 provider_options: None,
1426 }],
1427 provider_options: None,
1428 };
1429 rec_a.input.prompt = vec![mk("hello")];
1430 rec_b.input.prompt = vec![mk("hello"), mk("world")];
1431
1432 let matcher = PrefixMatcher::new("openai", "gpt-4o");
1433 let opts = GenerateTextOptions::default().into_call_options(vec![
1435 mk("hello"),
1436 mk("world"),
1437 mk("!"),
1438 ]);
1439 let recs = [rec_a, rec_b];
1440 let hit = matcher.r#match(&opts, &recs).unwrap();
1441 assert_eq!(hit.call_id, "tb");
1442 }
1443
1444 #[test]
1445 fn unmatched_returns_clear_error() {
1446 let recs = [openai_recording("t1", "ping", "pong", "stop")];
1447 let model = MockReplayModel::new("openai", "gpt-4o", vec![recs[0].clone()]);
1448 let rt = tokio::runtime::Runtime::new().unwrap();
1449 let err = rt.block_on(async {
1450 model
1451 .do_generate(&sample_options("nope", None))
1452 .await
1453 .unwrap_err()
1454 });
1455 assert!(err.to_string().contains("mock replay"), "{err}");
1456 }
1457
1458 #[test]
1459 fn mock_model_rebuilds_generate_result_from_openai_body() {
1460 let rec = openai_recording("t1", "ping", "pong", "stop");
1461 let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1462 let rt = tokio::runtime::Runtime::new().unwrap();
1463 let result = rt.block_on(async {
1464 model
1465 .do_generate(&sample_options("ping", None))
1466 .await
1467 .unwrap()
1468 });
1469 let Some(GenerateContent::Text { text, .. }) = result.content.first() else {
1470 panic!("expected text content");
1471 };
1472 assert_eq!(text, "pong");
1473 assert_eq!(result.finish_reason.unified, FinishReasonUnified::Stop);
1474 assert_eq!(result.usage.input_tokens.total, Some(5));
1475 assert_eq!(result.response.model_id.as_deref(), Some("gpt-4o"));
1476 }
1477
1478 #[test]
1479 fn mock_model_usage_overflow_errors() {
1480 let mut rec = openai_recording("t1", "ping", "pong", "stop");
1482 let body = serde_json::json!({
1483 "id": "chatcmpl-mock",
1484 "model": "gpt-4o",
1485 "choices": [{
1486 "index": 0,
1487 "message": { "role": "assistant", "content": "pong" },
1488 "finish_reason": "stop"
1489 }],
1490 "usage": { "prompt_tokens": 5_000_000_000_u64, "completion_tokens": 3, "total_tokens": 8 }
1491 });
1492 rec.exchanges[0].response.as_mut().unwrap().body = Some(body.to_string());
1493 let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1494 let rt = tokio::runtime::Runtime::new().unwrap();
1495 let err = rt.block_on(async {
1496 model
1497 .do_generate(&sample_options("ping", None))
1498 .await
1499 .unwrap_err()
1500 });
1501 assert!(matches!(err, AiMuxError::JsonParse(_)), "{err}");
1502 assert!(err.to_string().contains("overflows u32"), "{err}");
1503 }
1504
1505 #[test]
1506 fn mock_model_rebuilds_stream_from_sse() {
1507 let sse = "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"hel\"}}]}\n\n\
1508 data: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\n\
1509 data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n\
1510 data: [DONE]\n\n";
1511 let mut rec = openai_recording("t1", "ping", "x", "stop");
1512 rec.exchanges[0].response.as_mut().unwrap().body = Some(sse.to_string());
1513 rec.exchanges[0].response.as_mut().unwrap().stream_chunks = Some(4);
1514 let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1515 let rt = tokio::runtime::Runtime::new().unwrap();
1516 let parts: Vec<StreamPart> = rt.block_on(async {
1517 model
1518 .do_stream(&sample_options("ping", None))
1519 .await
1520 .unwrap()
1521 .stream
1522 .collect::<Vec<_>>()
1523 .await
1524 .into_iter()
1525 .map(|p| p.unwrap())
1526 .collect()
1527 });
1528 assert!(matches!(parts[0], StreamPart::StreamStart { .. }));
1529 let deltas: Vec<String> = parts
1530 .iter()
1531 .filter_map(|p| match p {
1532 StreamPart::TextDelta { delta, .. } => Some(delta.clone()),
1533 _ => None,
1534 })
1535 .collect();
1536 assert_eq!(deltas, vec!["hel", "lo"]);
1537 assert!(parts.iter().any(|p| matches!(p, StreamPart::Finish { .. })));
1538 }
1539
1540 #[test]
1541 fn non_openai_body_returns_unsupported() {
1542 let mut rec = openai_recording("t1", "ping", "x", "stop");
1545 rec.exchanges[0].response.as_mut().unwrap().body = Some("{\"foo\":\"bar\"}".to_string()); let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1547 let rt = tokio::runtime::Runtime::new().unwrap();
1548 let err = rt.block_on(async {
1549 model
1550 .do_generate(&sample_options("ping", None))
1551 .await
1552 .unwrap_err()
1553 });
1554 assert!(
1555 matches!(err, AiMuxError::UnsupportedFunctionality(_)),
1556 "{err}"
1557 );
1558 }
1559
1560 #[test]
1561 fn non_openai_stream_returns_unsupported() {
1562 let mut rec = openai_recording("t1", "ping", "x", "stop");
1564 rec.exchanges[0].response.as_mut().unwrap().body =
1565 Some("event: message_start\ndata: {\"type\":\"message_start\"}\n\n".to_string());
1566 rec.exchanges[0].response.as_mut().unwrap().stream_chunks = Some(1);
1567 let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1568 let rt = tokio::runtime::Runtime::new().unwrap();
1569 let err = rt.block_on(async {
1570 model
1571 .do_stream(&sample_options("ping", None))
1572 .await
1573 .unwrap_err()
1574 });
1575 assert!(
1576 matches!(err, AiMuxError::UnsupportedFunctionality(_)),
1577 "{err}"
1578 );
1579 }
1580
1581 #[test]
1582 fn mock_model_rebuilds_tool_calls_from_openai_body() {
1583 let mut rec = openai_recording("t1", "ping", "x", "tool_calls");
1585 let body = serde_json::json!({
1586 "id": "chatcmpl-mock",
1587 "model": "gpt-4o",
1588 "choices": [{
1589 "index": 0,
1590 "message": {
1591 "role": "assistant",
1592 "content": null,
1593 "tool_calls": [{
1594 "id": "call_abc",
1595 "type": "function",
1596 "function": { "name": "get_weather", "arguments": "{\"city\":\"SF\"}" }
1597 }]
1598 },
1599 "finish_reason": "tool_calls"
1600 }],
1601 "usage": { "prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8 }
1602 });
1603 rec.exchanges[0].response.as_mut().unwrap().body = Some(body.to_string());
1604 let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1605 let rt = tokio::runtime::Runtime::new().unwrap();
1606 let result = rt.block_on(async {
1607 model
1608 .do_generate(&sample_options("ping", None))
1609 .await
1610 .unwrap()
1611 });
1612 assert_eq!(result.content.len(), 1);
1613 let Some(GenerateContent::ToolCall {
1614 tool_call_id,
1615 tool_name,
1616 input,
1617 ..
1618 }) = result.content.first()
1619 else {
1620 panic!("expected ToolCall, got {:?}", result.content);
1621 };
1622 assert_eq!(tool_call_id, "call_abc");
1623 assert_eq!(tool_name, "get_weather");
1624 assert_eq!(input, &serde_json::json!(r#"{"city":"SF"}"#));
1625 assert_eq!(result.finish_reason.unified, FinishReasonUnified::ToolCalls);
1626 }
1627
1628 #[test]
1629 fn mock_model_tool_call_bad_json_arguments_falls_back_to_string() {
1630 let mut rec = openai_recording("t1", "ping", "x", "tool_calls");
1632 let body = serde_json::json!({
1633 "id": "chatcmpl-mock",
1634 "model": "gpt-4o",
1635 "choices": [{
1636 "index": 0,
1637 "message": {
1638 "role": "assistant",
1639 "content": null,
1640 "tool_calls": [{
1641 "id": "call_1",
1642 "type": "function",
1643 "function": { "name": "f", "arguments": "not-json{" }
1644 }]
1645 },
1646 "finish_reason": "tool_calls"
1647 }]
1648 });
1649 rec.exchanges[0].response.as_mut().unwrap().body = Some(body.to_string());
1650 let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1651 let rt = tokio::runtime::Runtime::new().unwrap();
1652 let result = rt.block_on(async {
1653 model
1654 .do_generate(&sample_options("ping", None))
1655 .await
1656 .unwrap()
1657 });
1658 let Some(GenerateContent::ToolCall { input, .. }) = result.content.first() else {
1659 panic!("expected ToolCall");
1660 };
1661 assert_eq!(input, &serde_json::json!("not-json{"));
1662 }
1663
1664 #[test]
1665 fn mock_model_rebuilds_stream_tool_calls() {
1666 let chunk1 = serde_json::json!({
1668 "id": "chatcmpl-1", "model": "gpt-4o",
1669 "choices": [{"index": 0, "delta": {"role": "assistant", "tool_calls": [
1670 {"index": 0, "id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": ""}}
1671 ]}}]
1672 });
1673 let chunk2 = serde_json::json!({
1674 "choices": [{"index": 0, "delta": {"tool_calls": [
1675 {"index": 0, "function": {"arguments": "{\"city\":\"SF\"}"}}
1676 ]}}]
1677 });
1678 let chunk3 = serde_json::json!({"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]});
1679 let sse = format!("data: {chunk1}\n\ndata: {chunk2}\n\ndata: {chunk3}\n\ndata: [DONE]\n\n");
1680 let mut rec = openai_recording("t1", "ping", "x", "stop");
1681 rec.exchanges[0].response.as_mut().unwrap().body = Some(sse);
1682 rec.exchanges[0].response.as_mut().unwrap().stream_chunks = Some(4);
1683 let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1684 let rt = tokio::runtime::Runtime::new().unwrap();
1685 let parts: Vec<StreamPart> = rt.block_on(async {
1686 model
1687 .do_stream(&sample_options("ping", None))
1688 .await
1689 .unwrap()
1690 .stream
1691 .collect::<Vec<_>>()
1692 .await
1693 .into_iter()
1694 .map(|p| p.unwrap())
1695 .collect()
1696 });
1697 assert!(parts.iter().any(|p| matches!(
1699 p,
1700 StreamPart::ResponseMetadata { id, model_id, .. }
1701 if id.as_deref() == Some("chatcmpl-1") && model_id.as_deref() == Some("gpt-4o")
1702 )));
1703 assert!(parts.iter().any(|p| matches!(
1705 p,
1706 StreamPart::ToolInputStart { tool_name, .. } if tool_name == "get_weather"
1707 )));
1708 let deltas: Vec<String> = parts
1710 .iter()
1711 .filter_map(|p| match p {
1712 StreamPart::ToolInputDelta { delta, .. } => Some(delta.clone()),
1713 _ => None,
1714 })
1715 .collect();
1716 assert_eq!(deltas, vec!["{\"city\":\"SF\"}"]);
1717 assert!(
1719 parts
1720 .iter()
1721 .any(|p| matches!(p, StreamPart::ToolInputEnd { .. }))
1722 );
1723 let calls: Vec<_> = parts
1724 .iter()
1725 .filter_map(|p| match p {
1726 StreamPart::ToolCall {
1727 tool_call_id,
1728 tool_name,
1729 input,
1730 ..
1731 } => Some((tool_call_id.clone(), tool_name.clone(), input.clone())),
1732 _ => None,
1733 })
1734 .collect();
1735 assert_eq!(calls.len(), 1);
1736 assert_eq!(calls[0].0, "call_1");
1737 assert_eq!(calls[0].1, "get_weather");
1738 assert_eq!(calls[0].2, serde_json::json!(r#"{"city":"SF"}"#));
1739 assert!(parts.iter().any(|p| matches!(
1741 p,
1742 StreamPart::Finish { finish_reason, .. }
1743 if finish_reason.unified == FinishReasonUnified::ToolCalls
1744 )));
1745 }
1746
1747 #[test]
1748 fn mock_model_rebuilds_stream_metadata_and_usage() {
1749 let chunk1 = serde_json::json!({
1752 "id": "chatcmpl-2", "model": "gpt-4o",
1753 "choices": [{"index": 0, "delta": {"content": "hi"}}]
1754 });
1755 let chunk2 = serde_json::json!({
1756 "id": "chatcmpl-2", "choices": [],
1757 "usage": {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}
1758 });
1759 let sse = format!("data: {chunk1}\n\ndata: {chunk2}\n\ndata: [DONE]\n\n");
1760 let mut rec = openai_recording("t1", "ping", "x", "stop");
1761 rec.exchanges[0].response.as_mut().unwrap().body = Some(sse);
1762 rec.exchanges[0].response.as_mut().unwrap().stream_chunks = Some(3);
1763 let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1764 let rt = tokio::runtime::Runtime::new().unwrap();
1765 let parts: Vec<StreamPart> = rt.block_on(async {
1766 model
1767 .do_stream(&sample_options("ping", None))
1768 .await
1769 .unwrap()
1770 .stream
1771 .collect::<Vec<_>>()
1772 .await
1773 .into_iter()
1774 .map(|p| p.unwrap())
1775 .collect()
1776 });
1777 assert!(parts.iter().any(|p| matches!(
1778 p,
1779 StreamPart::ResponseMetadata { id, model_id, .. }
1780 if id.as_deref() == Some("chatcmpl-2") && model_id.as_deref() == Some("gpt-4o")
1781 )));
1782 let deltas: Vec<String> = parts
1783 .iter()
1784 .filter_map(|p| match p {
1785 StreamPart::TextDelta { delta, .. } => Some(delta.clone()),
1786 _ => None,
1787 })
1788 .collect();
1789 assert_eq!(deltas, vec!["hi"]);
1790 let finish = parts
1791 .iter()
1792 .find_map(|p| match p {
1793 StreamPart::Finish { usage, .. } => Some(usage.clone()),
1794 _ => None,
1795 })
1796 .unwrap();
1797 assert_eq!(finish.input_tokens.total, Some(7));
1798 assert_eq!(finish.output_tokens.total, Some(2));
1799 }
1800
1801 #[test]
1802 fn last_response_skips_failed_attempt() {
1803 let mut rec = openai_recording("t1", "ping", "pong", "stop");
1805 rec.exchanges[0].response.as_mut().unwrap().status = 500;
1806 rec.exchanges[0].attempt = 0;
1807 let success_body = serde_json::json!({
1808 "id": "chatcmpl-ok", "model": "gpt-4o",
1809 "choices": [{"index":0,"message":{"role":"assistant","content":"pong"},"finish_reason":"stop"}],
1810 "usage": {"prompt_tokens":5,"completion_tokens":3,"total_tokens":8}
1811 });
1812 rec.exchanges.push(HttpExchange {
1813 step: None,
1814 attempt: 1,
1815 exchange_index: 1,
1816 request: HttpRecord {
1817 method: "post".into(),
1818 url: "https://api.openai.com/v1/chat/completions".into(),
1819 headers: vec![],
1820 body: Some("{}".into()),
1821 },
1822 response: Some(ResponseRecord {
1823 status: 200,
1824 headers: vec![],
1825 body: Some(success_body.to_string()),
1826 stream_chunks: None,
1827 ttfb_ms: None,
1828 }),
1829 timing: TimingRecord {
1830 latency_ms: 10,
1831 ttfb_ms: None,
1832 },
1833 error: None,
1834 finalized: true,
1835 });
1836 let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1837 let rt = tokio::runtime::Runtime::new().unwrap();
1838 let result = rt.block_on(async {
1839 model
1840 .do_generate(&sample_options("ping", None))
1841 .await
1842 .unwrap()
1843 });
1844 let Some(GenerateContent::Text { text, .. }) = result.content.first() else {
1845 panic!("expected text");
1846 };
1847 assert_eq!(text, "pong");
1848 assert_eq!(result.response.id.as_deref(), Some("chatcmpl-ok"));
1850 }
1851
1852 #[test]
1853 fn last_response_no_successful_response_errors() {
1854 let mut rec = openai_recording("t1", "ping", "pong", "stop");
1856 rec.exchanges[0].response.as_mut().unwrap().status = 500;
1857 let model = MockReplayModel::new("openai", "gpt-4o", vec![rec]);
1858 let rt = tokio::runtime::Runtime::new().unwrap();
1859 let err = rt.block_on(async {
1860 model
1861 .do_generate(&sample_options("ping", None))
1862 .await
1863 .unwrap_err()
1864 });
1865 assert!(err.to_string().contains("no successful"), "{err}");
1866 }
1867
1868 #[test]
1869 fn from_jsonl_loads_recordings() {
1870 let dir = std::env::temp_dir().join(format!("aimux-replay-jsonl-{}", std::process::id()));
1871 let _ = std::fs::remove_dir_all(&dir);
1872 std::fs::create_dir_all(&dir).unwrap();
1873 let path = dir.join("rec.jsonl");
1874 let rec = openai_recording("t1", "ping", "pong", "stop");
1875 std::fs::write(&path, serde_json::to_string(&rec).unwrap() + "\n").unwrap();
1876
1877 let model = MockReplayModel::from_jsonl(path.to_str().unwrap()).unwrap();
1878 assert_eq!(model.provider(), "openai");
1879 assert_eq!(model.model_id(), "gpt-4o");
1880 assert_eq!(model.recordings().len(), 1);
1881 let _ = std::fs::remove_dir_all(&dir);
1882 }
1883
1884 #[derive(Clone)]
1888 struct EchoModel {
1889 provider: &'static str,
1890 model_id: &'static str,
1891 }
1892
1893 impl EchoModel {
1894 fn new() -> Self {
1895 Self {
1896 provider: "openai",
1897 model_id: "gpt-4o",
1898 }
1899 }
1900 }
1901
1902 #[async_trait]
1903 impl LanguageModel for EchoModel {
1904 fn provider(&self) -> &str {
1905 self.provider
1906 }
1907 fn model_id(&self) -> &str {
1908 self.model_id
1909 }
1910 async fn do_generate(&self, options: &CallOptions) -> Result<GenerateResult, AiMuxError> {
1911 let text = options
1912 .prompt
1913 .iter()
1914 .filter_map(|m| match m.content.first() {
1915 Some(crate::content::ContentPart::Text { text, .. }) => Some(text.clone()),
1916 _ => None,
1917 })
1918 .collect::<Vec<_>>()
1919 .join(" ");
1920 Ok(GenerateResult {
1921 content: vec![GenerateContent::Text {
1922 text,
1923 provider_metadata: None,
1924 }],
1925 finish_reason: FinishReason {
1926 unified: FinishReasonUnified::Stop,
1927 raw: Some("stop".into()),
1928 },
1929 usage: Usage::default(),
1930 warnings: vec![],
1931 provider_metadata: None,
1932 response: ResponseMetadata::default(),
1933 request_body: None,
1934 response_headers: None,
1935 })
1936 }
1937 async fn do_stream(&self, _options: &CallOptions) -> Result<StreamResult, AiMuxError> {
1938 unreachable!("not used in tests")
1939 }
1940 }
1941
1942 fn optioned_recording(prompt_text: &str) -> Recording {
1945 let mut rec = openai_recording("t1", prompt_text, "pong", "stop");
1946 let mut call = sample_options(prompt_text, Some(0.7));
1947 call.max_output_tokens = Some(128);
1948 rec.input.options = serde_json::to_value(&call).unwrap();
1949 rec
1950 }
1951
1952 #[test]
1953 fn replay_with_model_rebuilds_input_and_resends() {
1954 let rec = optioned_recording("hello");
1955 let model = EchoModel::new();
1956 let rt = tokio::runtime::Runtime::new().unwrap();
1957 let result = rt.block_on(async { replay_with_model(&rec, &model, None).await.unwrap() });
1958 assert_eq!(result.text, "hello");
1959 }
1960
1961 #[test]
1962 fn replay_with_model_applies_overrides() {
1963 let rec = optioned_recording("hello");
1964 let model = EchoModel::new();
1965 let overrides = ReplayOverrides {
1966 prompt: Some(vec![
1967 crate::language_model_message::LanguageModelPromptMessage {
1968 role: crate::message::Role::User,
1969 content: vec![crate::content::ContentPart::Text {
1970 text: "overridden".into(),
1971 provider_options: None,
1972 }],
1973 provider_options: None,
1974 },
1975 ]),
1976 ..Default::default()
1977 };
1978 let rt = tokio::runtime::Runtime::new().unwrap();
1979 let result = rt.block_on(async {
1980 replay_with_model(&rec, &model, Some(&overrides))
1981 .await
1982 .unwrap()
1983 });
1984 assert_eq!(result.text, "overridden");
1985 }
1986
1987 #[test]
1988 fn replay_with_model_bad_input_options_errors() {
1989 let mut rec = openai_recording("t1", "hello", "pong", "stop");
1990 rec.input.options = serde_json::json!({ "not": "call-options" });
1991 let model = EchoModel::new();
1992 let rt = tokio::runtime::Runtime::new().unwrap();
1993 let err = rt.block_on(async { replay_with_model(&rec, &model, None).await.unwrap_err() });
1994 assert!(matches!(err, AiMuxError::JsonParse(_)), "{err}");
1995 }
1996}