1use crate::value::VmDictExt;
2use std::cell::RefCell;
3use std::collections::{BTreeMap, BTreeSet};
4use std::sync::atomic::{AtomicU64, Ordering};
5use std::sync::{Arc, LazyLock, Mutex, MutexGuard};
6
7use super::api::{LlmResult, ProviderTelemetry, RawProviderToolCall};
8use super::mock_store::{MockQueue, QueueMatch};
9use crate::orchestration::ToolCallRecord;
10use crate::value::{ErrorCategory, VmError, VmValue};
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum LlmReplayMode {
15 Off,
16 Record,
17 Replay,
18}
19
20#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
21enum CliLlmMockMode {
22 #[default]
23 Off,
24 Replay,
25 Record,
26}
27
28#[derive(Default)]
29struct CliLlmMockState {
30 mode: CliLlmMockMode,
31 queue: MockQueue,
32 recordings: Vec<LlmMock>,
33}
34
35static CLI_LLM_MOCK_NEXT_SCOPE: AtomicU64 = AtomicU64::new(1);
36static CLI_LLM_MOCK_SCOPES: LazyLock<Mutex<BTreeMap<u64, CliLlmMockState>>> =
37 LazyLock::new(|| Mutex::new(BTreeMap::new()));
38
39struct CliLlmMockLease(u64);
40
41impl Drop for CliLlmMockLease {
42 fn drop(&mut self) {
43 cli_llm_mock_scopes().remove(&self.0);
44 }
45}
46
47#[derive(Clone, Debug)]
52pub struct MockError {
53 pub category: ErrorCategory,
54 pub message: String,
55 pub status: Option<u16>,
56 pub kind: Option<String>,
57 pub reason: Option<String>,
58 pub retry_after_ms: Option<u64>,
63}
64
65impl MockError {
66 fn has_provider_envelope(&self) -> bool {
67 self.status.is_some() || self.kind.is_some() || self.reason.is_some()
68 }
69}
70
71pub(crate) fn build_mock_error(
72 category: Option<String>,
73 message: Option<String>,
74 status: Option<u16>,
75 kind: Option<String>,
76 reason: Option<String>,
77 retry_after_ms: Option<u64>,
78) -> Result<MockError, String> {
79 if retry_after_ms.is_some_and(|ms| ms > i64::MAX as u64) {
80 return Err("error.retry_after_ms must fit in a signed 64-bit integer".to_string());
81 }
82 let kind = match kind {
83 Some(value) if value.trim().is_empty() => None,
84 Some(value) => {
85 let normalized = value.trim().to_ascii_lowercase();
86 if super::api::LlmErrorKind::parse(&normalized).is_none() {
87 return Err(format!("unknown error kind `{value}`"));
88 }
89 Some(normalized)
90 }
91 None => None,
92 };
93 let reason = reason.and_then(|value| {
94 let trimmed = value.trim();
95 if trimmed.is_empty() {
96 None
97 } else {
98 Some(trimmed.to_string())
99 }
100 });
101 let category_was_provided = category.is_some();
102 let category = match category {
103 Some(value) if value.trim().is_empty() => {
104 return Err("error.category must not be empty".to_string());
105 }
106 Some(value) => {
107 let normalized = value.trim().to_ascii_lowercase();
108 let category = ErrorCategory::parse(&normalized);
109 if category.as_str() != normalized {
110 return Err(format!("unknown error category `{value}`"));
111 }
112 category
113 }
114 None => infer_mock_error_category(status, kind.as_deref(), reason.as_deref()),
115 };
116 if !category_was_provided && kind.is_none() && status.is_none() && reason.is_none() {
117 return Err(
118 "error.category is required unless error.status, error.kind, or error.reason is set"
119 .to_string(),
120 );
121 }
122 Ok(MockError {
123 category,
124 message: message.unwrap_or_else(|| {
125 default_mock_error_message(status, kind.as_deref(), reason.as_deref())
126 }),
127 status,
128 kind,
129 reason,
130 retry_after_ms,
131 })
132}
133
134pub(crate) fn validate_mock_error_status(status: i64) -> Result<u16, String> {
135 let status = u16::try_from(status)
136 .map_err(|_| "error.status must be an HTTP status code".to_string())?;
137 reqwest::StatusCode::from_u16(status)
138 .map_err(|_| "error.status must be an HTTP status code".to_string())?;
139 Ok(status)
140}
141
142fn infer_mock_error_category(
143 status: Option<u16>,
144 kind: Option<&str>,
145 reason: Option<&str>,
146) -> ErrorCategory {
147 if let Some(status) = status {
148 match status {
149 401 | 403 => return ErrorCategory::Auth,
150 404 | 410 => return ErrorCategory::NotFound,
151 408 | 504 | 522 | 524 => return ErrorCategory::Timeout,
152 429 => return ErrorCategory::RateLimit,
153 503 | 529 => return ErrorCategory::Overloaded,
154 500 | 502 => return ErrorCategory::ServerError,
155 _ => {}
156 }
157 }
158 if let Some(reason) = reason {
159 match reason {
160 "rate_limit" => return ErrorCategory::RateLimit,
161 "timeout" => return ErrorCategory::Timeout,
162 "network_error" | "transient_network" => return ErrorCategory::TransientNetwork,
163 "server_error" | "provider_error" | "provider_5xx" | "upstream_unavailable" => {
164 return ErrorCategory::ServerError;
165 }
166 "auth_failure" => return ErrorCategory::Auth,
167 "model_unavailable" => return ErrorCategory::NotFound,
168 _ => {}
169 }
170 }
171 if kind == Some("transient") {
172 return ErrorCategory::ServerError;
173 }
174 ErrorCategory::Generic
175}
176
177fn default_mock_error_message(
178 status: Option<u16>,
179 kind: Option<&str>,
180 reason: Option<&str>,
181) -> String {
182 match (status, kind, reason) {
183 (Some(status), Some(kind), Some(reason)) => {
184 format!("HTTP {status} mock LLM error ({kind}/{reason})")
185 }
186 (Some(status), _, Some(reason)) => format!("HTTP {status} mock LLM error ({reason})"),
187 (Some(status), _, _) => format!("HTTP {status} mock LLM error"),
188 (None, Some(kind), Some(reason)) => format!("mock LLM error ({kind}/{reason})"),
189 (None, Some(kind), None) => format!("mock LLM error ({kind})"),
190 (None, None, Some(reason)) => format!("mock LLM error ({reason})"),
191 (None, None, None) => String::new(),
192 }
193}
194
195pub const DEFAULT_MOCK_SCOPE: &str = "default";
199
200pub const KNOWN_MOCK_SCOPES: &[&str] = &[
205 DEFAULT_MOCK_SCOPE,
206 "agent.main",
207 "agent.input_guardrail",
208 "agent.scope_classifier",
209 "compaction",
210 "completion.judge",
211 "step.judge",
212];
213
214#[derive(Clone, Debug, PartialEq, Eq)]
219pub struct MockConsumptionReceipt {
220 pub requested_scope: String,
221 pub resolved_scope: String,
222 pub matched: bool,
223 pub id: String,
224 pub consume: String,
226 pub fell_through: bool,
227 pub remaining: usize,
228}
229
230impl MockConsumptionReceipt {
231 pub(crate) fn hit(
232 requested_scope: &str,
233 resolved_scope: &str,
234 mock: &LlmMock,
235 fell_through: bool,
236 remaining: usize,
237 ) -> Self {
238 Self {
239 requested_scope: requested_scope.to_string(),
240 resolved_scope: resolved_scope.to_string(),
241 matched: true,
242 id: mock.entry_id.clone(),
243 consume: consume_label(mock.sticky).to_string(),
244 fell_through,
245 remaining,
246 }
247 }
248
249 pub(crate) fn miss(requested_scope: &str, remaining: usize) -> Self {
250 Self {
251 requested_scope: requested_scope.to_string(),
252 resolved_scope: String::new(),
253 matched: false,
254 id: String::new(),
255 consume: String::new(),
256 fell_through: false,
257 remaining,
258 }
259 }
260}
261
262fn consume_label(sticky: bool) -> &'static str {
264 if sticky {
265 "sticky"
266 } else {
267 "once"
268 }
269}
270
271#[derive(Clone, Debug)]
272pub struct LlmMock {
273 pub text: String,
274 pub tool_calls: Vec<serde_json::Value>,
275 pub raw_tool_calls: Vec<RawProviderToolCall>,
276 pub match_pattern: Option<String>, pub scope: String,
281 pub entry_id: String,
285 pub sticky: bool,
289 pub input_tokens: Option<i64>,
290 pub output_tokens: Option<i64>,
291 pub cache_read_tokens: Option<i64>,
292 pub cache_write_tokens: Option<i64>,
293 pub thinking: Option<String>,
294 pub thinking_summary: Option<String>,
295 pub stop_reason: Option<String>,
296 pub model: String,
297 pub provider: Option<String>,
298 pub blocks: Option<Vec<serde_json::Value>>,
299 pub logprobs: Vec<serde_json::Value>,
300 pub error: Option<MockError>,
303 pub stream_chunks: Vec<String>,
310}
311
312#[derive(Clone, Debug, Default)]
317pub struct LlmMockFixture {
318 pub schema_version: u32,
319 pub strict_scopes: bool,
320 pub mocks: Vec<LlmMock>,
321 pub warnings: Vec<String>,
322}
323
324#[derive(Clone, Debug, PartialEq, Eq)]
327pub(crate) struct LlmMockFixtureReceipt {
328 pub schema_version: u32,
329 pub strict_scopes: bool,
330 pub count: usize,
331 pub scopes: Vec<String>,
332 pub warnings: Vec<String>,
333}
334
335pub const MAX_MOCK_SCHEMA_VERSION: u32 = 1;
337
338#[derive(Clone)]
339pub(crate) struct LlmMockCall {
340 pub mock_scope: String,
341 pub api_mode: String,
342 pub messages: Vec<serde_json::Value>,
343 pub system: Option<String>,
344 pub tools: Option<Vec<serde_json::Value>>,
345 pub provider_tools: Option<Vec<serde_json::Value>>,
346 pub tool_choice: Option<serde_json::Value>,
347 pub output_format: serde_json::Value,
348 pub thinking: serde_json::Value,
349 pub previous_response_id: Option<String>,
350 pub store: Option<bool>,
351 pub background: Option<bool>,
352 pub truncation: Option<String>,
353 pub compact: Option<bool>,
354 pub include: Option<Vec<String>>,
355 pub max_tool_calls: Option<i64>,
356 pub prefill: Option<String>,
357}
358
359type LlmMockScope = (
360 MockQueue,
361 Vec<LlmMockCall>,
362 BTreeSet<String>,
363 Vec<MockConsumptionReceipt>,
364);
365
366#[derive(Default)]
367struct LlmMockState {
368 builtin_queue: MockQueue,
369 calls: Vec<LlmMockCall>,
370 prompt_cache: BTreeSet<String>,
371 scopes: Vec<LlmMockScope>,
372 receipts: Vec<MockConsumptionReceipt>,
373 cli_scope: Option<Arc<CliLlmMockLease>>,
374}
375
376#[derive(Clone, Default)]
383pub(crate) struct LlmMockContext(Arc<Mutex<LlmMockState>>);
384
385impl LlmMockContext {
386 pub(crate) fn for_new_vm() -> Self {
387 let context = Self::default();
388 context.lock().cli_scope = current_cli_llm_mock_lease();
389 context
390 }
391
392 fn lock(&self) -> MutexGuard<'_, LlmMockState> {
393 self.0
394 .lock()
395 .unwrap_or_else(|poisoned| poisoned.into_inner())
396 }
397}
398
399thread_local! {
400 static LLM_REPLAY_MODE: RefCell<LlmReplayMode> = const { RefCell::new(LlmReplayMode::Off) };
401 static LLM_FIXTURE_DIR: RefCell<String> = const { RefCell::new(String::new()) };
402 static TOOL_RECORDINGS: RefCell<Vec<ToolCallRecord>> = const { RefCell::new(Vec::new()) };
403 static LLM_MOCK_CONTEXT: RefCell<LlmMockContext> = RefCell::new(LlmMockContext::default());
404 static LLM_MOCK_STREAM_CHUNKS: RefCell<Option<Vec<String>>> = const { RefCell::new(None) };
409}
410
411pub(crate) fn swap_llm_mock_context(next: LlmMockContext) -> LlmMockContext {
412 LLM_MOCK_CONTEXT.with(|slot| std::mem::replace(&mut *slot.borrow_mut(), next))
413}
414
415pub(crate) fn current_llm_mock_context() -> LlmMockContext {
416 LLM_MOCK_CONTEXT.with(|slot| slot.borrow().clone())
417}
418
419fn with_mock_state<T>(f: impl FnOnce(&LlmMockState) -> T) -> T {
420 let context = current_llm_mock_context();
421 let state = context.lock();
422 f(&state)
423}
424
425fn with_mock_state_mut<T>(f: impl FnOnce(&mut LlmMockState) -> T) -> T {
426 let context = current_llm_mock_context();
427 let mut state = context.lock();
428 f(&mut state)
429}
430
431pub(crate) fn set_mock_stream_chunks(chunks: Option<Vec<String>>) {
435 LLM_MOCK_STREAM_CHUNKS.with(|slot| *slot.borrow_mut() = chunks);
436}
437
438pub(crate) fn take_mock_stream_chunks() -> Option<Vec<String>> {
442 LLM_MOCK_STREAM_CHUNKS.with(|slot| slot.borrow_mut().take())
443}
444
445fn cli_llm_mock_scopes() -> MutexGuard<'static, BTreeMap<u64, CliLlmMockState>> {
446 CLI_LLM_MOCK_SCOPES
447 .lock()
448 .unwrap_or_else(|poisoned| poisoned.into_inner())
449}
450
451fn next_cli_llm_mock_scope_id() -> u64 {
452 CLI_LLM_MOCK_NEXT_SCOPE.fetch_add(1, Ordering::Relaxed)
453}
454
455pub(crate) fn current_cli_llm_mock_scope() -> Option<u64> {
456 current_cli_llm_mock_lease().map(|scope| scope.0)
457}
458
459fn current_cli_llm_mock_lease() -> Option<Arc<CliLlmMockLease>> {
460 with_mock_state(|state| state.cli_scope.clone())
461}
462
463fn install_cli_llm_mock_scope(state: CliLlmMockState) {
464 clear_cli_llm_mock_mode();
465 let scope = next_cli_llm_mock_scope_id();
466 cli_llm_mock_scopes().insert(scope, state);
467 with_mock_state_mut(|state| state.cli_scope = Some(Arc::new(CliLlmMockLease(scope))));
468}
469
470#[cfg(test)]
473pub(crate) fn push_llm_mock(mock: LlmMock) {
474 with_mock_state_mut(|state| state.builtin_queue.push_v0(mock));
475}
476
477pub(crate) fn push_inline_llm_mock(mock: LlmMock) -> Result<(), String> {
481 with_mock_state_mut(|state| {
482 let queue = &mut state.builtin_queue;
483 if queue.schema_version() > 0 {
484 return Err(
485 "cannot append harness.llm.mock_enqueue() entries to an active versioned fixture; clear or load one complete document"
486 .to_string(),
487 );
488 }
489 queue.push_v0(mock);
490 Ok(())
491 })
492}
493
494pub(crate) fn install_builtin_llm_mock_fixture(fixture: LlmMockFixture) -> LlmMockFixtureReceipt {
498 let queue = MockQueue::from_fixture(fixture);
499 let receipt = LlmMockFixtureReceipt {
500 schema_version: queue.schema_version(),
501 strict_scopes: queue.strict_scopes(),
502 count: queue.count(),
503 scopes: queue.scopes(),
504 warnings: queue.warnings().to_vec(),
505 };
506 with_mock_state_mut(|state| state.builtin_queue = queue);
507 receipt
508}
509
510pub(crate) fn get_llm_mock_calls() -> Vec<LlmMockCall> {
511 with_mock_state(|state| state.calls.clone())
512}
513
514pub(crate) fn get_llm_mock_receipts() -> Vec<MockConsumptionReceipt> {
516 with_mock_state(|state| state.receipts.clone())
517}
518
519fn record_mock_receipt(session_id: Option<&str>, receipt: MockConsumptionReceipt) {
520 if receipt.matched {
521 if let Some(session_id) = session_id.filter(|id| !id.is_empty()) {
522 super::agent_runtime::emit_agent_event_sync(
523 &crate::agent_events::AgentEvent::TypedCheckpoint {
524 session_id: session_id.to_string(),
525 checkpoint: serde_json::json!({
526 "kind": "llm_mock_fixture_consumption",
527 "schema": "harn.llm_mock_fixture_consumption.v1",
528 "id": receipt.id,
529 "requested_scope": receipt.requested_scope,
530 "resolved_scope": receipt.resolved_scope,
531 "consume": receipt.consume,
532 "fell_through": receipt.fell_through,
533 "remaining": receipt.remaining,
534 }),
535 },
536 );
537 }
538 }
539 with_mock_state_mut(|state| state.receipts.push(receipt));
540}
541
542pub(crate) fn builtin_llm_mock_snapshot() -> serde_json::Value {
543 with_mock_state(|state| {
544 let queue = &state.builtin_queue;
545 serde_json::json!({
546 "schema": "harn.llm_mock_fixture_queue.v1",
547 "schema_version": queue.schema_version(),
548 "strict_scopes": queue.strict_scopes(),
549 "queue_remaining": queue.queue_remaining(),
550 "warnings": queue.warnings(),
551 })
552 })
553}
554
555pub(crate) fn builtin_llm_mock_active() -> bool {
556 with_mock_state(|state| state.builtin_queue.is_active())
557}
558
559pub(crate) fn builtin_llm_mock_strict_scopes() -> bool {
560 with_mock_state(|state| state.builtin_queue.strict_scopes())
561}
562
563pub(crate) fn reset_llm_mock_state() {
564 with_mock_state_mut(|state| {
565 state.cli_scope = None;
566 state.builtin_queue = MockQueue::default();
567 state.calls.clear();
568 state.prompt_cache.clear();
569 state.scopes.clear();
570 state.receipts.clear();
571 });
572}
573
574pub(crate) fn push_llm_mock_scope() {
579 with_mock_state_mut(|state| {
580 let fixture = std::mem::take(&mut state.builtin_queue);
581 let calls = std::mem::take(&mut state.calls);
582 let cache = std::mem::take(&mut state.prompt_cache);
583 let receipts = std::mem::take(&mut state.receipts);
584 state.scopes.push((fixture, calls, cache, receipts));
585 });
586}
587
588pub(crate) fn pop_llm_mock_scope() -> bool {
594 with_mock_state_mut(|state| match state.scopes.pop() {
595 Some((fixture, calls, cache, receipts)) => {
596 state.builtin_queue = fixture;
597 state.calls = calls;
598 state.prompt_cache = cache;
599 state.receipts = receipts;
600 true
601 }
602 None => false,
603 })
604}
605
606pub fn clear_cli_llm_mock_mode() {
607 with_mock_state_mut(|state| state.cli_scope = None);
608}
609
610pub fn install_cli_llm_mocks(mocks: Vec<LlmMock>) {
611 install_cli_llm_mock_scope(CliLlmMockState {
612 mode: CliLlmMockMode::Replay,
613 queue: MockQueue::from_fixture(LlmMockFixture {
614 schema_version: 0,
615 strict_scopes: false,
616 mocks,
617 warnings: Vec::new(),
618 }),
619 recordings: Vec::new(),
620 });
621}
622
623pub fn install_cli_llm_mock_fixture(fixture: LlmMockFixture) {
625 install_cli_llm_mock_scope(CliLlmMockState {
626 mode: CliLlmMockMode::Replay,
627 queue: MockQueue::from_fixture(fixture),
628 recordings: Vec::new(),
629 });
630}
631
632pub fn enable_cli_llm_mock_recording() {
633 install_cli_llm_mock_scope(CliLlmMockState {
634 mode: CliLlmMockMode::Record,
635 queue: MockQueue::default(),
636 recordings: Vec::new(),
637 });
638}
639
640pub fn take_cli_llm_recordings() -> Vec<LlmMock> {
641 let Some(scope) = current_cli_llm_mock_scope() else {
642 return Vec::new();
643 };
644 cli_llm_mock_scopes()
645 .get_mut(&scope)
646 .map(|state| std::mem::take(&mut state.recordings))
647 .unwrap_or_default()
648}
649
650pub(crate) fn cli_llm_mock_replay_active() -> bool {
651 cli_llm_mock_replay_active_for_scope(current_cli_llm_mock_scope())
652}
653
654pub(crate) fn cli_llm_mock_replay_active_for_scope(scope: Option<u64>) -> bool {
655 let Some(scope) = scope else {
656 return false;
657 };
658 cli_llm_mock_scopes()
659 .get(&scope)
660 .is_some_and(|state| state.mode == CliLlmMockMode::Replay)
661}
662
663fn record_llm_mock_call(request: &super::api::LlmRequestPayload) {
664 with_mock_state_mut(|state| {
665 state.calls.push(LlmMockCall {
666 mock_scope: request
667 .mock_scope
668 .as_deref()
669 .unwrap_or(DEFAULT_MOCK_SCOPE)
670 .to_string(),
671 api_mode: request.api_mode.as_str().to_string(),
672 messages: request.messages.clone(),
673 system: request.system.clone(),
674 tools: request.native_tools.clone(),
675 provider_tools: if request.provider_tools.is_empty() {
676 None
677 } else {
678 Some(request.provider_tools.clone())
679 },
680 tool_choice: request.tool_choice.clone(),
681 output_format: serde_json::to_value(&request.output_format).unwrap_or_else(|_| {
682 serde_json::json!({
683 "kind": "text"
684 })
685 }),
686 thinking: serde_json::to_value(&request.thinking).unwrap_or_else(|_| {
687 serde_json::json!({
688 "mode": "disabled"
689 })
690 }),
691 previous_response_id: request.previous_response_id.clone(),
692 store: request.store,
693 background: request.background,
694 truncation: request.truncation.clone(),
695 compact: request.compact,
696 include: request.include.clone(),
697 max_tool_calls: request.max_tool_calls,
698 prefill: request.prefill.clone(),
699 });
700 });
701}
702
703fn build_mock_result(mock: &LlmMock, last_msg_len: usize) -> LlmResult {
705 let effective_text = if !mock.stream_chunks.is_empty() && mock.text.is_empty() {
710 mock.stream_chunks.concat()
711 } else {
712 mock.text.clone()
713 };
714 set_mock_stream_chunks(if mock.stream_chunks.is_empty() {
715 None
716 } else {
717 Some(mock.stream_chunks.clone())
718 });
719 let mock = &LlmMock {
720 text: effective_text,
721 ..mock.clone()
722 };
723 let (tool_calls, blocks) = if let Some(blocks) = &mock.blocks {
724 (mock.tool_calls.clone(), blocks.clone())
725 } else {
726 let mut blocks = Vec::new();
727
728 if !mock.text.is_empty() {
729 blocks.push(serde_json::json!({
730 "type": "output_text",
731 "text": mock.text,
732 "visibility": "public",
733 }));
734 }
735
736 let mut tool_calls = Vec::new();
737 for (i, tc) in mock.tool_calls.iter().enumerate() {
738 let id = format!("mock_call_{}", i + 1);
739 let name = tc.get("name").and_then(|n| n.as_str()).unwrap_or("unknown");
740 let arguments = tc
741 .get("arguments")
742 .cloned()
743 .unwrap_or(serde_json::json!({}));
744 tool_calls.push(serde_json::json!({
745 "id": id,
746 "type": "tool_call",
747 "name": name,
748 "arguments": arguments,
749 }));
750 blocks.push(serde_json::json!({
751 "type": "tool_call",
752 "id": id,
753 "name": name,
754 "arguments": arguments,
755 "visibility": "internal",
756 }));
757 }
758
759 (tool_calls, blocks)
760 };
761
762 LlmResult {
763 text_projection: None,
764 served_fast: false,
765 text: mock.text.clone(),
766 raw_tool_calls: if mock.raw_tool_calls.is_empty() {
767 Vec::new()
768 } else {
769 mock.raw_tool_calls.clone()
770 },
771 tool_calls,
772 input_tokens: mock.input_tokens.unwrap_or(last_msg_len as i64),
773 output_tokens: mock.output_tokens.unwrap_or(30),
774 cache_read_tokens: mock.cache_read_tokens.unwrap_or(0),
775 cache_write_tokens: mock.cache_write_tokens.unwrap_or(0),
776 cache_supported: true,
777 model: mock.model.clone(),
778 provider: mock.provider.clone().unwrap_or_else(|| "mock".to_string()),
779 thinking: mock.thinking.clone(),
780 thinking_summary: mock.thinking_summary.clone(),
781 stop_reason: mock.stop_reason.clone(),
782 blocks,
783 logprobs: mock.logprobs.clone(),
784 telemetry: ProviderTelemetry::default(),
785 }
786}
787
788fn collect_mock_match_strings(value: &serde_json::Value, out: &mut Vec<String>) {
792 match value {
793 serde_json::Value::String(text) if !text.is_empty() => out.push(text.clone()),
794 serde_json::Value::String(_) => {}
795 serde_json::Value::Array(items) => {
796 for item in items {
797 collect_mock_match_strings(item, out);
798 }
799 }
800 serde_json::Value::Object(map) => {
801 for value in map.values() {
802 collect_mock_match_strings(value, out);
803 }
804 }
805 _ => {}
806 }
807}
808
809fn mock_match_text(messages: &[serde_json::Value]) -> String {
810 let mut parts = Vec::new();
811 for message in messages {
812 collect_mock_match_strings(message, &mut parts);
813 }
814 parts.join("\n")
815}
816
817fn mock_last_prompt_text(messages: &[serde_json::Value]) -> String {
818 for message in messages.iter().rev() {
819 let Some(content) = message.get("content") else {
820 continue;
821 };
822 let mut parts = Vec::new();
823 collect_mock_match_strings(content, &mut parts);
824 let text = parts.join("\n");
825 if !text.trim().is_empty() {
826 return text;
827 }
828 }
829 String::new()
830}
831
832fn mock_prompt_cache_key(
833 model: &str,
834 messages: &[serde_json::Value],
835 system: Option<&str>,
836 mock_scope: &str,
837) -> String {
838 serde_json::to_string(&serde_json::json!({
839 "model": model,
840 "system": system,
841 "messages": messages,
842 "mock_scope": mock_scope,
843 }))
844 .unwrap_or_default()
845}
846
847fn apply_mock_prompt_cache(result: &mut LlmResult, cache_key: &str) {
848 if result.cache_read_tokens > 0 || result.cache_write_tokens > 0 {
849 return;
850 }
851 let cache_tokens = result.input_tokens.max(0);
852 if cache_tokens == 0 {
853 return;
854 }
855 let cache_hit = with_mock_state_mut(|state| {
856 if state.prompt_cache.contains(cache_key) {
857 true
858 } else {
859 state.prompt_cache.insert(cache_key.to_string());
860 false
861 }
862 });
863 if cache_hit {
864 result.cache_read_tokens = cache_tokens;
865 } else {
866 result.cache_write_tokens = cache_tokens;
867 }
868}
869
870fn mock_error_to_vm_error(err: &MockError) -> VmError {
874 let message = mock_error_message(err);
875 if err.has_provider_envelope() {
876 let classified = super::api::classify_llm_error(err.category.clone(), &message);
877 let mut dict = BTreeMap::new();
878 dict.put_str("category", err.category.as_str());
879 dict.put_str(
880 "kind",
881 err.kind
882 .as_deref()
883 .unwrap_or_else(|| classified.kind.as_str()),
884 );
885 dict.put_str(
886 "reason",
887 err.reason
888 .as_deref()
889 .unwrap_or_else(|| classified.reason.as_str()),
890 );
891 dict.put_str("message", message);
892 if let Some(status) = err.status {
893 dict.insert("status".to_string(), VmValue::Int(i64::from(status)));
894 }
895 if let Some(retry_after_ms) = err.retry_after_ms {
896 dict.insert(
897 "retry_after_ms".to_string(),
898 VmValue::Int(retry_after_ms as i64),
899 );
900 }
901 return VmError::Thrown(VmValue::dict(dict));
902 }
903
904 VmError::CategorizedError {
905 message,
906 category: err.category.clone(),
907 }
908}
909
910fn mock_error_message(err: &MockError) -> String {
911 let Some(ms) = err.retry_after_ms else {
915 return err.message.clone();
916 };
917 if err.has_provider_envelope() {
918 return err.message.clone();
919 }
920 let secs = (ms as f64 / 1000.0).max(0.0);
921 let sep = if err.message.is_empty() || err.message.ends_with('\n') {
922 ""
923 } else {
924 "\n"
925 };
926 format!("{}{sep}retry-after: {secs}\n", err.message)
927}
928
929struct ScopedMatch {
932 outcome: Result<LlmResult, VmError>,
933 receipt: MockConsumptionReceipt,
934}
935
936fn build_scoped_match(selected: QueueMatch, match_text: &str) -> ScopedMatch {
937 let QueueMatch { mock, receipt } = selected;
938 let outcome = match &mock.error {
939 Some(err) => Err(mock_error_to_vm_error(err)),
940 None => Ok(build_mock_result(&mock, match_text.len())),
941 };
942 ScopedMatch { outcome, receipt }
943}
944
945fn try_match_builtin_mock(scope: &str, match_text: &str) -> Option<ScopedMatch> {
946 with_mock_state_mut(|state| {
947 state
948 .builtin_queue
949 .match_request(scope, match_text)
950 .map(|selected| build_scoped_match(selected, match_text))
951 })
952}
953
954fn try_match_cli_mock(
955 cli_scope: Option<u64>,
956 scope: &str,
957 match_text: &str,
958) -> Option<ScopedMatch> {
959 let cli_scope = cli_scope?;
960 let mut scopes = cli_llm_mock_scopes();
961 let state = scopes.get_mut(&cli_scope)?;
962 if state.mode != CliLlmMockMode::Replay {
963 return None;
964 }
965 state
966 .queue
967 .match_request(scope, match_text)
968 .map(|selected| build_scoped_match(selected, match_text))
969}
970
971pub(crate) fn record_cli_llm_result(request: &super::api::LlmRequestPayload, result: &LlmResult) {
972 record_unified_tape_llm_call(result);
973 let Some(scope) = request.cli_llm_mock_scope else {
974 return;
975 };
976 let mut scopes = cli_llm_mock_scopes();
977 let Some(state) = scopes.get_mut(&scope) else {
978 return;
979 };
980 if state.mode != CliLlmMockMode::Record {
981 return;
982 }
983 state.recordings.push(LlmMock {
984 text: result.text.clone(),
985 tool_calls: result.tool_calls.clone(),
986 raw_tool_calls: result.raw_tool_calls.clone(),
987 match_pattern: None,
988 scope: request
989 .mock_scope
990 .clone()
991 .unwrap_or_else(|| DEFAULT_MOCK_SCOPE.to_string()),
992 entry_id: String::new(),
993 sticky: false,
994 input_tokens: Some(result.input_tokens),
995 output_tokens: Some(result.output_tokens),
996 cache_read_tokens: Some(result.cache_read_tokens),
997 cache_write_tokens: Some(result.cache_write_tokens),
998 thinking: result.thinking.clone(),
999 thinking_summary: result.thinking_summary.clone(),
1000 stop_reason: result.stop_reason.clone(),
1001 model: result.model.clone(),
1002 provider: Some(result.provider.clone()),
1003 blocks: Some(result.blocks.clone()),
1004 logprobs: result.logprobs.clone(),
1005 error: None,
1006 stream_chunks: Vec::new(),
1007 });
1008}
1009
1010fn record_unified_tape_llm_call(result: &LlmResult) {
1017 if crate::testbench::tape::active_recorder().is_none() {
1018 return;
1019 }
1020 let response_json = serde_json::to_vec(result).unwrap_or_else(|_| Vec::new());
1021 let request_digest = with_mock_state(|state| state.calls.last().cloned())
1022 .map(|call| {
1023 let mut request = serde_json::Map::new();
1024 request.insert("messages".to_string(), serde_json::json!(call.messages));
1025 request.insert("system".to_string(), serde_json::json!(call.system));
1026 request.insert("tools".to_string(), serde_json::json!(call.tools));
1027 request.insert(
1028 "tool_choice".to_string(),
1029 serde_json::json!(call.tool_choice),
1030 );
1031 request.insert("thinking".to_string(), serde_json::json!(call.thinking));
1032 if call.mock_scope != DEFAULT_MOCK_SCOPE {
1033 request.insert("mock_scope".to_string(), serde_json::json!(call.mock_scope));
1034 }
1035 request.insert("model".to_string(), serde_json::json!(result.model));
1036 if call.api_mode != "chat_completions" {
1037 request.insert("api_mode".to_string(), serde_json::json!(call.api_mode));
1038 }
1039 if call.provider_tools.is_some() {
1040 request.insert(
1041 "provider_tools".to_string(),
1042 serde_json::json!(call.provider_tools),
1043 );
1044 }
1045 if call
1046 .output_format
1047 .get("kind")
1048 .and_then(|value| value.as_str())
1049 != Some("text")
1050 {
1051 request.insert(
1052 "output_format".to_string(),
1053 serde_json::json!(call.output_format),
1054 );
1055 }
1056 if call.previous_response_id.is_some() {
1057 request.insert(
1058 "previous_response_id".to_string(),
1059 serde_json::json!(call.previous_response_id),
1060 );
1061 }
1062 if call.store.is_some() {
1063 request.insert("store".to_string(), serde_json::json!(call.store));
1064 }
1065 if call.background.is_some() {
1066 request.insert("background".to_string(), serde_json::json!(call.background));
1067 }
1068 if call.truncation.is_some() {
1069 request.insert("truncation".to_string(), serde_json::json!(call.truncation));
1070 }
1071 if call.compact.is_some() {
1072 request.insert("compact".to_string(), serde_json::json!(call.compact));
1073 }
1074 if call.include.is_some() {
1075 request.insert("include".to_string(), serde_json::json!(call.include));
1076 }
1077 if call.max_tool_calls.is_some() {
1078 request.insert(
1079 "max_tool_calls".to_string(),
1080 serde_json::json!(call.max_tool_calls),
1081 );
1082 }
1083 if call.prefill.is_some() {
1084 request.insert("prefill".to_string(), serde_json::json!(call.prefill));
1085 }
1086 let serialized = crate::canonical_json::to_vec(&serde_json::Value::Object(request));
1087 crate::testbench::tape::content_hash(&serialized)
1088 })
1089 .unwrap_or_else(|| {
1090 crate::testbench::tape::content_hash(result.text.as_bytes())
1093 });
1094 crate::testbench::tape::with_active_recorder(|recorder| {
1095 let response = recorder.payload_from_bytes(response_json);
1096 Some(crate::testbench::tape::TapeRecordKind::LlmCall {
1097 request_digest,
1098 response,
1099 })
1100 });
1101}
1102
1103fn unmatched_cli_prompt_error(match_text: &str) -> VmError {
1104 let mut snippet: String = match_text.chars().take(200).collect();
1105 if match_text.chars().count() > 200 {
1106 snippet.push_str("...");
1107 }
1108 VmError::Runtime(format!("No --llm-mock fixture matched prompt: {snippet:?}"))
1109}
1110
1111fn unmatched_builtin_prompt_error(match_text: &str) -> VmError {
1112 let mut snippet: String = match_text.chars().take(200).collect();
1113 if match_text.chars().count() > 200 {
1114 snippet.push_str("...");
1115 }
1116 VmError::Runtime(format!(
1117 "No llm_mock fixture matched prompt in a strict scope: {snippet:?}"
1118 ))
1119}
1120
1121pub fn set_replay_mode(mode: LlmReplayMode, fixture_dir: &str) {
1123 LLM_REPLAY_MODE.with(|v| *v.borrow_mut() = mode);
1124 LLM_FIXTURE_DIR.with(|v| *v.borrow_mut() = fixture_dir.to_string());
1125}
1126
1127pub(crate) fn get_replay_mode() -> LlmReplayMode {
1128 LLM_REPLAY_MODE.with(|v| *v.borrow())
1129}
1130
1131pub(crate) fn get_fixture_dir() -> String {
1132 LLM_FIXTURE_DIR.with(|v| v.borrow().clone())
1133}
1134
1135pub(crate) fn fixture_hash(
1137 model: &str,
1138 messages: &[serde_json::Value],
1139 system: Option<&str>,
1140 mock_scope: Option<&str>,
1141) -> String {
1142 use std::hash::{Hash, Hasher};
1143 let mut hasher = std::collections::hash_map::DefaultHasher::new();
1144 model.hash(&mut hasher);
1145 serde_json::to_string(messages)
1147 .unwrap_or_default()
1148 .hash(&mut hasher);
1149 system.hash(&mut hasher);
1150 if mock_scope.is_some_and(|scope| scope != DEFAULT_MOCK_SCOPE) {
1151 mock_scope.hash(&mut hasher);
1152 }
1153 format!("{:016x}", hasher.finish())
1154}
1155
1156pub(crate) fn fixture_hash_for_request(request: &super::api::LlmRequestPayload) -> String {
1157 fixture_hash(
1158 &request.model,
1159 &request.messages,
1160 request.system.as_deref(),
1161 request.mock_scope.as_deref(),
1162 )
1163}
1164
1165pub(crate) fn save_fixture(hash: &str, result: &LlmResult) {
1166 let dir = get_fixture_dir();
1167 if dir.is_empty() {
1168 return;
1169 }
1170 let _ = std::fs::create_dir_all(&dir);
1171 let path = format!("{dir}/{hash}.json");
1172 let json = serde_json::json!({
1173 "text": result.text,
1174 "tool_calls": result.tool_calls,
1175 "raw_tool_calls": result.raw_tool_calls,
1176 "input_tokens": result.input_tokens,
1177 "output_tokens": result.output_tokens,
1178 "cache_read_tokens": result.cache_read_tokens,
1179 "cache_write_tokens": result.cache_write_tokens,
1180 "model": result.model,
1181 "provider": result.provider,
1182 "thinking": result.thinking,
1183 "thinking_summary": result.thinking_summary,
1184 "stop_reason": result.stop_reason,
1185 "blocks": result.blocks,
1186 "logprobs": result.logprobs,
1187 });
1188 let _ = std::fs::write(
1189 &path,
1190 serde_json::to_string_pretty(&json).unwrap_or_default(),
1191 );
1192}
1193
1194pub(crate) fn load_fixture(hash: &str) -> Option<LlmResult> {
1195 let dir = get_fixture_dir();
1196 if dir.is_empty() {
1197 return None;
1198 }
1199 let path = format!("{dir}/{hash}.json");
1200 let content = std::fs::read_to_string(&path).ok()?;
1201 let json: serde_json::Value = serde_json::from_str(&content).ok()?;
1202 Some(LlmResult {
1203 text_projection: None,
1204 served_fast: false,
1205 text: json["text"].as_str().unwrap_or("").to_string(),
1206 tool_calls: json["tool_calls"].as_array().cloned().unwrap_or_default(),
1207 raw_tool_calls: RawProviderToolCall::array_from_value(&json["raw_tool_calls"]).ok()?,
1208 input_tokens: json["input_tokens"].as_i64().unwrap_or(0),
1209 output_tokens: json["output_tokens"].as_i64().unwrap_or(0),
1210 cache_read_tokens: json["cache_read_tokens"].as_i64().unwrap_or(0),
1211 cache_write_tokens: json["cache_write_tokens"]
1212 .as_i64()
1213 .or_else(|| json["cache_creation_input_tokens"].as_i64())
1214 .unwrap_or(0),
1215 cache_supported: json["cache_supported"].as_bool().unwrap_or(true),
1216 model: json["model"].as_str().unwrap_or("").to_string(),
1217 provider: json["provider"].as_str().unwrap_or("mock").to_string(),
1218 thinking: json["thinking"].as_str().map(|s| s.to_string()),
1219 thinking_summary: json["thinking_summary"].as_str().map(|s| s.to_string()),
1220 stop_reason: json["stop_reason"].as_str().map(|s| s.to_string()),
1221 blocks: json["blocks"].as_array().cloned().unwrap_or_default(),
1222 logprobs: json["logprobs"].as_array().cloned().unwrap_or_default(),
1223 telemetry: serde_json::from_value(json["telemetry"].clone()).unwrap_or_default(),
1224 })
1225}
1226
1227fn mock_required_args(tool_schema: &serde_json::Value) -> serde_json::Value {
1231 let mut args = serde_json::Map::new();
1232 let input_schema = tool_schema
1236 .get("input_schema")
1237 .or_else(|| tool_schema.get("inputSchema"))
1238 .or_else(|| {
1239 tool_schema
1240 .get("function")
1241 .and_then(|f| f.get("parameters"))
1242 })
1243 .or_else(|| tool_schema.get("parameters"));
1244 let Some(schema) = input_schema else {
1245 return serde_json::Value::Object(args);
1246 };
1247 let required: std::collections::BTreeSet<String> = schema
1248 .get("required")
1249 .and_then(|r| r.as_array())
1250 .map(|arr| {
1251 arr.iter()
1252 .filter_map(|v| v.as_str().map(|s| s.to_string()))
1253 .collect()
1254 })
1255 .unwrap_or_default();
1256 if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) {
1257 for (name, prop) in props {
1258 if !required.contains(name) {
1259 continue;
1260 }
1261 let ty = prop
1262 .get("type")
1263 .and_then(|t| t.as_str())
1264 .unwrap_or("string");
1265 let placeholder = match ty {
1266 "integer" => serde_json::json!(0),
1267 "number" => serde_json::json!(0.0),
1268 "boolean" => serde_json::json!(false),
1269 "array" => serde_json::json!([]),
1270 "object" => serde_json::json!({}),
1271 _ => serde_json::json!(""),
1272 };
1273 args.insert(name.clone(), placeholder);
1274 }
1275 }
1276 serde_json::Value::Object(args)
1277}
1278
1279fn mock_tool_name(tool: &serde_json::Value) -> Option<&str> {
1280 tool.get("name")
1281 .or_else(|| {
1282 tool.get("function")
1283 .and_then(|function| function.get("name"))
1284 })
1285 .and_then(|name| name.as_str())
1286}
1287
1288fn mock_auto_tool_candidate(tools: &[serde_json::Value]) -> Option<&serde_json::Value> {
1289 tools
1290 .iter()
1291 .find(|tool| mock_tool_name(tool) != Some("agent_await_resumption"))
1292}
1293
1294pub(crate) fn mock_llm_response(
1299 request: &super::api::LlmRequestPayload,
1300) -> Result<LlmResult, VmError> {
1301 record_llm_mock_call(request);
1302 set_mock_stream_chunks(None);
1305
1306 let messages = &request.messages;
1307 let system = request.system.as_deref();
1308 let match_text = mock_match_text(messages);
1309 let prompt_text = mock_last_prompt_text(messages);
1310 let requested_scope = request.mock_scope.as_deref().unwrap_or(DEFAULT_MOCK_SCOPE);
1311 let cache_key = mock_prompt_cache_key(&request.model, messages, system, requested_scope);
1312
1313 if let Some(matched) =
1314 try_match_cli_mock(request.cli_llm_mock_scope, requested_scope, &match_text)
1315 {
1316 record_mock_receipt(request.session_id.as_deref(), matched.receipt);
1317 return matched.outcome.map(|mut result| {
1318 if request.cache {
1319 apply_mock_prompt_cache(&mut result, &cache_key);
1320 }
1321 result
1322 });
1323 }
1324
1325 if let Some(matched) = try_match_builtin_mock(requested_scope, &match_text) {
1326 record_mock_receipt(request.session_id.as_deref(), matched.receipt);
1327 return matched.outcome.map(|mut result| {
1328 if request.cache {
1329 apply_mock_prompt_cache(&mut result, &cache_key);
1330 }
1331 result
1332 });
1333 }
1334
1335 if cli_llm_mock_replay_active_for_scope(request.cli_llm_mock_scope) || builtin_llm_mock_active()
1338 {
1339 let receipt = if cli_llm_mock_replay_active_for_scope(request.cli_llm_mock_scope) {
1340 let scopes = cli_llm_mock_scopes();
1341 scopes
1342 .get(&request.cli_llm_mock_scope.unwrap())
1343 .map(|state| state.queue.miss_receipt(requested_scope))
1344 .unwrap_or_else(|| MockConsumptionReceipt::miss(requested_scope, 0))
1345 } else {
1346 with_mock_state(|state| state.builtin_queue.miss_receipt(requested_scope))
1347 };
1348 record_mock_receipt(request.session_id.as_deref(), receipt);
1349 }
1350
1351 if cli_llm_mock_replay_active_for_scope(request.cli_llm_mock_scope) {
1352 return Err(unmatched_cli_prompt_error(&match_text));
1353 }
1354 if builtin_llm_mock_strict_scopes() {
1355 return Err(unmatched_builtin_prompt_error(&match_text));
1356 }
1357
1358 if let Some(tools) = request.native_tools.as_deref() {
1361 if let Some(first_tool) = mock_auto_tool_candidate(tools) {
1362 let tool_name = mock_tool_name(first_tool).unwrap_or("unknown");
1363 let mock_args = mock_required_args(first_tool);
1364 let mut result = LlmResult {
1365 text_projection: None,
1366 served_fast: false,
1367 text: String::new(),
1368 tool_calls: vec![serde_json::json!({
1369 "id": "mock_call_1",
1370 "type": "tool_call",
1371 "name": tool_name,
1372 "arguments": mock_args
1373 })],
1374 raw_tool_calls: Vec::new(),
1375 input_tokens: prompt_text.len() as i64,
1376 output_tokens: 20,
1377 cache_read_tokens: 0,
1378 cache_write_tokens: 0,
1379 cache_supported: true,
1380 model: request.model.clone(),
1381 provider: "mock".to_string(),
1382 thinking: None,
1383 thinking_summary: None,
1384 stop_reason: None,
1385 blocks: vec![serde_json::json!({
1386 "type": "tool_call",
1387 "id": "mock_call_1",
1388 "name": tool_name,
1389 "arguments": mock_args,
1390 "visibility": "internal",
1391 })],
1392 logprobs: Vec::new(),
1393 telemetry: ProviderTelemetry::default(),
1394 };
1395 if request.cache {
1396 apply_mock_prompt_cache(&mut result, &cache_key);
1397 }
1398 return Ok(result);
1399 }
1400 }
1401
1402 let tagged_done = system.is_some_and(|s| s.contains("<done>"));
1407
1408 let prose_body = if prompt_text.is_empty() {
1409 "Mock LLM response".to_string()
1410 } else {
1411 let word_count = prompt_text.split_whitespace().count();
1412 format!(
1413 "Mock response to {word_count}-word prompt: {}",
1414 prompt_text.chars().take(100).collect::<String>()
1415 )
1416 };
1417 let response = if tagged_done {
1418 format!("<assistant_prose>{prose_body}</assistant_prose>\n<done>##DONE##</done>")
1419 } else {
1420 prose_body
1421 };
1422
1423 let mut result = LlmResult {
1424 text_projection: None,
1425 served_fast: false,
1426 text: response.clone(),
1427 tool_calls: vec![],
1428 raw_tool_calls: Vec::new(),
1429 input_tokens: prompt_text.len() as i64,
1430 output_tokens: 30,
1431 cache_read_tokens: 0,
1432 cache_write_tokens: 0,
1433 cache_supported: true,
1434 model: request.model.clone(),
1435 provider: "mock".to_string(),
1436 thinking: None,
1437 thinking_summary: None,
1438 stop_reason: None,
1439 blocks: vec![serde_json::json!({
1440 "type": "output_text",
1441 "text": response,
1442 "visibility": "public",
1443 })],
1444 logprobs: Vec::new(),
1445 telemetry: ProviderTelemetry::default(),
1446 };
1447 if request.cache {
1448 apply_mock_prompt_cache(&mut result, &cache_key);
1449 }
1450 Ok(result)
1451}
1452
1453pub fn drain_tool_recordings() -> Vec<ToolCallRecord> {
1455 TOOL_RECORDINGS.with(|v| std::mem::take(&mut *v.borrow_mut()))
1456}
1457
1458#[cfg(test)]
1459#[path = "mock_tests.rs"]
1460mod tests;