Skip to main content

everruns_llmsim/
lib.rs

1#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))]
2#![deny(missing_docs)]
3
4//! Deterministic, offline LLM simulation for
5//! [Everruns](https://everruns.com) agents and runtimes.
6//!
7//! `everruns-llmsim` implements the provider contracts from
8//! [`everruns-contracts`](https://docs.rs/everruns-contracts) with configurable
9//! fixed, echo, sequence, and scripted responses. It runs in process without
10//! credentials or network access and supports deterministic tool calls,
11//! injected failures, latency controls, and request capture.
12//!
13//! Framework applications can use `everruns::Model::simulated` without naming
14//! this crate. Advanced hosts and tests can configure the driver directly:
15//!
16//! ```
17//! use everruns_llmsim::{LlmSimConfig, LlmSimDriver};
18//!
19//! let driver = LlmSimDriver::new(LlmSimConfig::fixed("Hello."));
20//! ```
21
22mod demo_scripts;
23mod model_names;
24pub mod realtime;
25mod scenarios;
26
27pub use demo_scripts::{
28    auditor_demo_script, guarded_bash_demo_script, monitor_demo_script, session_tasks_demo_script,
29};
30pub use scenarios::builtin_scenario_names;
31#[cfg(feature = "host")]
32mod runtime_ext;
33pub mod speech;
34
35#[cfg(feature = "host")]
36pub use runtime_ext::{LLMSIM_MODEL_ID, LLMSIM_PROVIDER, LlmSimRuntimeExt, llm_sim_provider};
37
38use async_trait::async_trait;
39use futures::StreamExt;
40use futures::stream;
41use std::sync::Arc;
42use std::sync::atomic::{AtomicUsize, Ordering};
43type SessionSequenceCounters =
44    std::collections::HashMap<String, (Arc<AtomicUsize>, Arc<AtomicUsize>)>;
45
46use everruns_contracts::driver_registry::{
47    BoxedChatDriver, ChatDriver, DriverDescriptor, DriverId, DriverRegistry, LlmCallConfig,
48    LlmCompletionMetadata, LlmResponseStream, LlmStreamEvent, Message, MessageRole,
49};
50use everruns_contracts::error::{AgentLoopError, Result};
51use everruns_contracts::tool_types::ToolCall;
52use llmsim::generator::{LoremGenerator, ResponseGenerator};
53use llmsim::latency::LatencyProfile;
54use llmsim::openai::{ChatCompletionRequest, Message as SimMessage, Role, Usage};
55use llmsim::script::auto_tool_call_id;
56use llmsim::stream::TokenStreamBuilder;
57
58// ============================================================================
59// Configuration Types
60// ============================================================================
61
62/// Configuration for the LlmSim driver.
63#[derive(Debug, Clone)]
64pub struct LlmSimConfig {
65    /// Response generation configuration
66    pub response: ResponseConfig,
67    /// Optional tool calls to include in responses
68    pub tool_calls: Option<ToolCallConfig>,
69    /// Enable latency simulation (default: false for fast tests)
70    pub simulate_latency: bool,
71    /// Model name to report in metadata
72    pub model_name: String,
73    /// Optional delay before responding (TTFT - time to first token).
74    /// This is useful for testing cancellation scenarios where we need a
75    /// predictable time window to cancel an active turn before completion.
76    pub response_delay: Option<std::time::Duration>,
77    /// Optional response ID to include in completion metadata.
78    /// Enables testing `previous_response_id` chaining.
79    pub response_id: Option<String>,
80    /// Optional capture sink for the per-call `reasoning_effort` (EVE-595).
81    /// When set, every `chat_completion_stream` call appends the effort it saw
82    /// in `LlmCallConfig`, in call order. Tests use this to assert that a
83    /// mid-turn effort change is observed by subsequent LLM steps.
84    pub effort_capture: Option<Arc<std::sync::Mutex<Vec<Option<String>>>>>,
85    /// Optional capture sink for the provider-visible messages of each call.
86    /// When set, every `chat_completion_stream` call appends the exact
87    /// `Message` slice it received, in call order. Tests use this to assert
88    /// which messages actually reach the provider after context assembly and
89    /// message filtering (e.g. Infinity Context history trimming).
90    pub message_capture: Option<Arc<std::sync::Mutex<Vec<Vec<Message>>>>>,
91    /// Context window to report for the simulated model, in tokens. A small
92    /// window makes proactive compaction run in long-session benches and
93    /// tests; `None` reports none, as before.
94    pub context_window: Option<usize>,
95}
96
97impl Default for LlmSimConfig {
98    fn default() -> Self {
99        Self {
100            response: ResponseConfig::Fixed("Hello! I'm a simulated LLM response.".to_string()),
101            tool_calls: None,
102            simulate_latency: false,
103            model_name: "llmsim-model".to_string(),
104            response_delay: None,
105            response_id: None,
106            effort_capture: None,
107            message_capture: None,
108            context_window: None,
109        }
110    }
111}
112
113impl LlmSimConfig {
114    /// Create a new config with a fixed response
115    pub fn fixed(response: impl Into<String>) -> Self {
116        Self {
117            response: ResponseConfig::Fixed(response.into()),
118            ..Default::default()
119        }
120    }
121
122    /// Create a new config that echoes user input
123    pub fn echo() -> Self {
124        Self {
125            response: ResponseConfig::Echo,
126            ..Default::default()
127        }
128    }
129
130    /// Create a new config with lorem ipsum text
131    pub fn lorem(target_tokens: usize) -> Self {
132        Self {
133            response: ResponseConfig::Lorem { target_tokens },
134            ..Default::default()
135        }
136    }
137
138    /// Create a new config with a sequence of responses
139    pub fn sequence(responses: Vec<String>) -> Self {
140        Self {
141            response: ResponseConfig::Sequence(responses),
142            ..Default::default()
143        }
144    }
145
146    /// Create a new config that replays scripted turns in order.
147    pub fn scripted(turns: Vec<SimTurn>) -> Self {
148        Self {
149            response: ResponseConfig::Scripted {
150                turns,
151                on_exhausted: OnExhausted::default(),
152            },
153            ..Default::default()
154        }
155    }
156
157    /// Set the behavior when a scripted response config exhausts its turns.
158    pub fn with_on_exhausted(mut self, mode: OnExhausted) -> Self {
159        if let ResponseConfig::Scripted { on_exhausted, .. } = &mut self.response {
160            *on_exhausted = mode;
161        }
162        self
163    }
164
165    /// Add tool calls to the response.
166    pub fn with_tool_calls(mut self, tool_calls: Vec<ToolCall>) -> Self {
167        self.tool_calls = Some(ToolCallConfig::Fixed(tool_calls));
168        self
169    }
170
171    /// Add a sequence of tool calls (different per call).
172    pub fn with_tool_call_sequence(mut self, sequences: Vec<Vec<ToolCall>>) -> Self {
173        self.tool_calls = Some(ToolCallConfig::Sequence(sequences));
174        self
175    }
176
177    /// Report a context window of `tokens` for the simulated model.
178    pub fn with_context_window(mut self, tokens: usize) -> Self {
179        self.context_window = Some(tokens);
180        self
181    }
182
183    /// Enable latency simulation.
184    pub fn with_latency(mut self) -> Self {
185        self.simulate_latency = true;
186        self
187    }
188
189    /// Set model name for metadata.
190    pub fn with_model(mut self, model: impl Into<String>) -> Self {
191        self.model_name = model.into();
192        self
193    }
194
195    /// Set a delay before responding (TTFT - time to first token).
196    /// This creates a predictable time window for testing cancellation scenarios.
197    pub fn with_response_delay(mut self, delay: std::time::Duration) -> Self {
198        self.response_delay = Some(delay);
199        self
200    }
201
202    /// Set a response ID to include in completion metadata (for testing chaining)
203    pub fn with_response_id(mut self, id: impl Into<String>) -> Self {
204        self.response_id = Some(id.into());
205        self
206    }
207
208    /// Set a shared capture sink for the per-call `reasoning_effort` (EVE-595).
209    /// Every `chat_completion_stream` call appends the effort it observed in
210    /// `LlmCallConfig`, in call order.
211    pub fn with_effort_capture(
212        mut self,
213        capture: Arc<std::sync::Mutex<Vec<Option<String>>>>,
214    ) -> Self {
215        self.effort_capture = Some(capture);
216        self
217    }
218
219    /// Set a shared capture sink for the provider-visible messages of each call.
220    /// Every `chat_completion_stream` call appends the exact `Message` slice
221    /// it received, in call order.
222    pub fn with_message_capture(
223        mut self,
224        capture: Arc<std::sync::Mutex<Vec<Vec<Message>>>>,
225    ) -> Self {
226        self.message_capture = Some(capture);
227        self
228    }
229
230    /// Create a new config that returns an error (for testing error handling)
231    pub fn error(message: impl Into<String>) -> Self {
232        Self {
233            response: ResponseConfig::Error(message.into()),
234            ..Default::default()
235        }
236    }
237
238    /// Create a new config that returns a model-not-available error
239    pub fn model_not_available() -> Self {
240        Self {
241            response: ResponseConfig::ModelNotAvailable,
242            ..Default::default()
243        }
244    }
245}
246
247/// Response generation configuration.
248#[derive(Debug, Clone)]
249pub enum ResponseConfig {
250    /// Return a fixed response
251    Fixed(String),
252    /// Echo back the last user message with a prefix
253    Echo,
254    /// Generate lorem ipsum text with target token count
255    Lorem {
256        /// Approximate number of generated tokens.
257        target_tokens: usize,
258    },
259    /// Return responses from a sequence (cycles when exhausted)
260    Sequence(Vec<String>),
261    /// Replay scripted assistant turns for multi-turn agent scenario tests.
262    Scripted {
263        /// Ordered turns to replay.
264        turns: Vec<SimTurn>,
265        /// Behavior after all turns have been consumed.
266        on_exhausted: OnExhausted,
267    },
268    /// Empty response (useful for tool-only responses)
269    Empty,
270    /// Simulate an error (useful for testing error handling)
271    Error(String),
272    /// Simulate a model-not-available error
273    ModelNotAvailable,
274}
275
276/// A single scripted assistant turn.
277#[derive(Debug, Clone, PartialEq)]
278pub enum SimTurn {
279    /// Plain assistant text response.
280    Assistant(String),
281    /// One or more tool calls in a single assistant turn.
282    ToolCalls(Vec<SimToolCall>),
283    /// Mixed assistant text and tool calls in the same turn.
284    Mixed {
285        /// Assistant text emitted before the tool calls.
286        text: String,
287        /// Tool calls emitted in the same assistant turn.
288        tool_calls: Vec<SimToolCall>,
289    },
290    /// Simulate an API/transport error on this turn.
291    Error(SimError),
292    /// Return a stream that never produces an event.
293    StreamStall,
294}
295
296/// A single tool call inside a scripted turn.
297#[derive(Debug, Clone, PartialEq)]
298pub struct SimToolCall {
299    /// Tool name.
300    pub name: String,
301    /// JSON arguments passed to the tool.
302    pub arguments: serde_json::Value,
303    /// Optional stable tool-call id; generated deterministically when absent.
304    pub id: Option<String>,
305}
306
307/// Error to inject for a scripted turn.
308#[derive(Debug, Clone, PartialEq)]
309pub enum SimError {
310    /// Provider rate limit.
311    RateLimit,
312    /// Provider timeout.
313    Timeout,
314    /// Transport failure.
315    Transport,
316    /// Provider overload.
317    Overloaded,
318    /// Authentication failure.
319    Authentication,
320    /// Provider quota exhaustion.
321    QuotaExhausted,
322    /// Unsupported model id.
323    UnsupportedModel(String),
324    /// Invalid provider response.
325    InvalidResponse(String),
326    /// Other injected LLM error.
327    Other(String),
328}
329
330impl SimError {
331    fn message(&self) -> String {
332        match self {
333            SimError::RateLimit => "Rate limit exceeded. Please retry after some time.".to_string(),
334            SimError::Timeout => "Request timed out".to_string(),
335            SimError::Transport => "Transport connection failed".to_string(),
336            SimError::Overloaded => "Provider overloaded".to_string(),
337            SimError::Authentication => "Invalid provider credentials".to_string(),
338            SimError::QuotaExhausted => "Provider quota exhausted".to_string(),
339            SimError::UnsupportedModel(model) => format!("Model not available: {model}"),
340            SimError::InvalidResponse(message) | SimError::Other(message) => message.clone(),
341        }
342    }
343
344    fn agent_error(&self) -> AgentLoopError {
345        use everruns_contracts::error::LlmErrorKind;
346
347        match self {
348            SimError::RateLimit => {
349                AgentLoopError::llm_kind(LlmErrorKind::RateLimited, self.message())
350            }
351            SimError::Timeout | SimError::Transport | SimError::Overloaded => {
352                AgentLoopError::llm_kind(LlmErrorKind::Unavailable, self.message())
353            }
354            SimError::Other(_) => AgentLoopError::llm_kind(LlmErrorKind::Other, self.message()),
355            SimError::Authentication => {
356                AgentLoopError::llm_kind(LlmErrorKind::Authentication, self.message())
357            }
358            SimError::QuotaExhausted => {
359                AgentLoopError::llm_kind(LlmErrorKind::QuotaExhausted, self.message())
360            }
361            SimError::UnsupportedModel(model) => AgentLoopError::model_not_available(model),
362            SimError::InvalidResponse(_) => {
363                AgentLoopError::llm_kind(LlmErrorKind::InvalidRequest, self.message())
364            }
365        }
366    }
367}
368
369/// Behavior when a scripted config has consumed all turns.
370#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
371pub enum OnExhausted {
372    /// Repeat the last turn forever.
373    #[default]
374    RepeatLast,
375    /// Return an error when the script is exhausted.
376    Error,
377    /// Cycle through the script from the start.
378    Loop,
379}
380
381/// Tool call configuration.
382#[derive(Debug, Clone)]
383pub enum ToolCallConfig {
384    /// Always return these tool calls
385    Fixed(Vec<ToolCall>),
386    /// Return tool calls from a sequence (cycles when exhausted)
387    Sequence(Vec<Vec<ToolCall>>),
388    /// Conditionally return tool calls based on message content
389    Conditional {
390        /// Patterns to match against user message
391        patterns: Vec<ToolCallPattern>,
392    },
393}
394
395/// Pattern for conditional tool calls
396#[derive(Debug, Clone)]
397pub struct ToolCallPattern {
398    /// Substring to match in user message
399    pub contains: String,
400    /// Tool calls to return when pattern matches
401    pub tool_calls: Vec<ToolCall>,
402}
403
404impl ToolCallPattern {
405    /// Match user messages containing `contains` and return `tool_calls`.
406    pub fn new(contains: impl Into<String>, tool_calls: Vec<ToolCall>) -> Self {
407        Self {
408            contains: contains.into(),
409            tool_calls,
410        }
411    }
412}
413
414fn materialize_scripted_tool_calls(
415    turn_index: usize,
416    calls: Vec<SimToolCall>,
417) -> Option<Vec<ToolCall>> {
418    if calls.is_empty() {
419        return None;
420    }
421
422    Some(
423        calls
424            .into_iter()
425            .enumerate()
426            .map(|(call_index, call)| ToolCall {
427                id: call
428                    .id
429                    .unwrap_or_else(|| auto_tool_call_id(turn_index, call_index)),
430                name: call.name,
431                arguments: call.arguments,
432            })
433            .collect(),
434    )
435}
436
437// ============================================================================
438// Driver Implementation
439// ============================================================================
440
441/// Deterministic LLM simulator driver.
442///
443/// This driver generates simulated responses based on configuration.
444/// It's intended for unit and integration tests where you need
445/// deterministic or configurable LLM behavior.
446///
447/// # Example
448///
449/// ```ignore
450/// use everruns_llmsim::{LlmSimDriver, LlmSimConfig};
451///
452/// // Simple fixed response
453/// let driver = LlmSimDriver::new(LlmSimConfig::fixed("Hello!"));
454///
455/// // With tool calls
456/// let driver = LlmSimDriver::new(
457///     LlmSimConfig::fixed("Let me check that for you.")
458///         .with_tool_calls(vec![ToolCall { ... }])
459/// );
460///
461/// // Sequence of responses for multi-turn tests
462/// let driver = LlmSimDriver::new(
463///     LlmSimConfig::sequence(vec![
464///         "First response".to_string(),
465///         "Second response".to_string(),
466///     ])
467/// );
468/// ```
469#[derive(Clone)]
470pub struct LlmSimDriver {
471    config: LlmSimConfig,
472    /// Counter for sequence-based responses
473    response_counter: Arc<AtomicUsize>,
474    /// Counter for sequence-based tool calls
475    tool_call_counter: Arc<AtomicUsize>,
476    /// Attempt counts for scenario steps that fail first
477    scenario_attempts: scenarios::Attempts,
478    // Registered Agents share one driver; sequence state belongs to each session.
479    session_counters: Arc<std::sync::Mutex<SessionSequenceCounters>>,
480}
481
482struct GeneratedTurn {
483    text: String,
484    tool_calls: Option<Vec<ToolCall>>,
485    stream_stall: bool,
486}
487
488impl LlmSimDriver {
489    /// Create a new driver with the given configuration
490    pub fn new(config: LlmSimConfig) -> Self {
491        Self {
492            config,
493            response_counter: Arc::new(AtomicUsize::new(0)),
494            tool_call_counter: Arc::new(AtomicUsize::new(0)),
495            scenario_attempts: scenarios::Attempts::default(),
496            session_counters: Arc::default(),
497        }
498    }
499
500    /// Create a driver with default configuration (fixed response)
501    pub fn default_driver() -> Self {
502        Self::new(LlmSimConfig::default())
503    }
504
505    /// Generate response text based on configuration
506    fn generate_response(&self, messages: &[Message], counter: &AtomicUsize) -> String {
507        match &self.config.response {
508            ResponseConfig::Fixed(text) => text.clone(),
509
510            ResponseConfig::Echo => {
511                // Find last user message and echo it
512                let last_user = messages
513                    .iter()
514                    .rev()
515                    .find(|m| m.role == MessageRole::User)
516                    .map(|m| m.content_as_text())
517                    .unwrap_or_default();
518                format!("Echo: {}", last_user)
519            }
520
521            ResponseConfig::Lorem { target_tokens } => {
522                let generator = LoremGenerator::new(*target_tokens);
523                let request = self.to_chat_request(messages);
524                generator.generate(&request)
525            }
526
527            ResponseConfig::Sequence(responses) => {
528                if responses.is_empty() {
529                    return String::new();
530                }
531                let idx = counter.fetch_add(1, Ordering::SeqCst);
532                responses[idx % responses.len()].clone()
533            }
534
535            ResponseConfig::Empty => String::new(),
536
537            // Error/ModelNotAvailable cases should never be reached; checked in chat_completion_stream
538            ResponseConfig::Error(_)
539            | ResponseConfig::ModelNotAvailable
540            | ResponseConfig::Scripted { .. } => {
541                unreachable!("Special configs handled in chat_completion_stream")
542            }
543        }
544    }
545
546    /// Get tool calls based on configuration
547    fn get_tool_calls(&self, messages: &[Message], counter: &AtomicUsize) -> Option<Vec<ToolCall>> {
548        match &self.config.tool_calls {
549            None => None,
550
551            Some(ToolCallConfig::Fixed(calls)) => {
552                if calls.is_empty() {
553                    None
554                } else {
555                    Some(calls.clone())
556                }
557            }
558
559            Some(ToolCallConfig::Sequence(sequences)) => {
560                if sequences.is_empty() {
561                    return None;
562                }
563                let idx = counter.fetch_add(1, Ordering::SeqCst);
564                let calls = &sequences[idx % sequences.len()];
565                if calls.is_empty() {
566                    None
567                } else {
568                    Some(calls.clone())
569                }
570            }
571
572            Some(ToolCallConfig::Conditional { patterns }) => {
573                // Scan user messages newest-first and use the first one that
574                // matches a pattern. Looking only at the very last user message
575                // is brittle: an injected user-role notification (e.g. a
576                // background task's terminal wake-up) can land after the
577                // triggering prompt and mask it, even though content-keyed
578                // patterns are meant to make scheduling order irrelevant.
579                // Newest-first honours the most recent matching intent while
580                // skipping interleaved non-matching notifications.
581                for message in messages.iter().rev() {
582                    if message.role != MessageRole::User {
583                        continue;
584                    }
585                    let text = message.content_as_text();
586                    if let Some(pattern) = patterns.iter().find(|p| text.contains(&p.contains)) {
587                        return if pattern.tool_calls.is_empty() {
588                            None
589                        } else {
590                            Some(pattern.tool_calls.clone())
591                        };
592                    }
593                }
594                None
595            }
596        }
597    }
598
599    fn generate_turn(
600        &self,
601        messages: &[Message],
602        response: &AtomicUsize,
603        tools: &AtomicUsize,
604    ) -> Result<GeneratedTurn> {
605        if let ResponseConfig::Scripted {
606            turns,
607            on_exhausted,
608        } = &self.config.response
609        {
610            return self.generate_scripted_turn(turns, *on_exhausted, response);
611        }
612
613        Ok(GeneratedTurn {
614            text: self.generate_response(messages, response),
615            tool_calls: self.get_tool_calls(messages, tools),
616            stream_stall: false,
617        })
618    }
619
620    fn generate_scripted_turn(
621        &self,
622        turns: &[SimTurn],
623        on_exhausted: OnExhausted,
624        counter: &AtomicUsize,
625    ) -> Result<GeneratedTurn> {
626        if turns.is_empty() {
627            return Err(AgentLoopError::config(
628                "llmsim scripted config must contain at least one turn",
629            ));
630        }
631
632        let turn_index = counter.fetch_add(1, Ordering::SeqCst);
633        let turn = if turn_index < turns.len() {
634            turns[turn_index].clone()
635        } else {
636            match on_exhausted {
637                OnExhausted::RepeatLast => turns[turns.len() - 1].clone(),
638                OnExhausted::Loop => turns[turn_index % turns.len()].clone(),
639                OnExhausted::Error => {
640                    return Err(AgentLoopError::config("llmsim scripted config exhausted"));
641                }
642            }
643        };
644
645        match turn {
646            SimTurn::Assistant(text) => Ok(GeneratedTurn {
647                text,
648                tool_calls: None,
649                stream_stall: false,
650            }),
651            SimTurn::ToolCalls(calls) => Ok(GeneratedTurn {
652                text: String::new(),
653                tool_calls: materialize_scripted_tool_calls(turn_index, calls),
654                stream_stall: false,
655            }),
656            SimTurn::Mixed { text, tool_calls } => Ok(GeneratedTurn {
657                text,
658                tool_calls: materialize_scripted_tool_calls(turn_index, tool_calls),
659                stream_stall: false,
660            }),
661            SimTurn::Error(error) => Err(error.agent_error()),
662            SimTurn::StreamStall => Ok(GeneratedTurn {
663                text: String::new(),
664                tool_calls: None,
665                stream_stall: true,
666            }),
667        }
668    }
669
670    /// Convert Message to llmsim ChatCompletionRequest
671    fn to_chat_request(&self, messages: &[Message]) -> ChatCompletionRequest {
672        let sim_messages: Vec<SimMessage> = messages
673            .iter()
674            .map(|m| {
675                let role = match m.role {
676                    MessageRole::System => Role::System,
677                    MessageRole::User => Role::User,
678                    MessageRole::Assistant => Role::Assistant,
679                    MessageRole::Tool => Role::Tool,
680                };
681                SimMessage {
682                    role,
683                    content: Some(m.content_as_text().into()),
684                    name: None,
685                    tool_calls: None,
686                    tool_call_id: m.tool_call_id.clone(),
687                }
688            })
689            .collect();
690
691        ChatCompletionRequest {
692            model: self.config.model_name.clone(),
693            messages: sim_messages,
694            temperature: None,
695            top_p: None,
696            n: None,
697            max_tokens: None,
698            max_completion_tokens: None,
699            stream: true,
700            stop: None,
701            presence_penalty: None,
702            frequency_penalty: None,
703            logit_bias: None,
704            user: None,
705            tools: None,
706            tool_choice: None,
707            seed: None,
708            response_format: None,
709            reasoning_effort: None,
710        }
711    }
712
713    /// Latency for a request: `-realistic` names (see [`model_names`]), else
714    /// `fast()` for `-latency` names or `simulate_latency`, else `instant()`.
715    fn resolve_latency_profile(&self, model_name: &str) -> LatencyProfile {
716        if let Some(profile) = model_names::realistic_latency(model_name) {
717            profile
718        } else if self.config.simulate_latency || model_name.contains("-latency") {
719            LatencyProfile::fast()
720        } else {
721            LatencyProfile::instant()
722        }
723    }
724
725    /// Estimate token count for text
726    fn estimate_tokens(text: &str) -> u32 {
727        // Simple estimation: ~4 chars per token
728        (text.len() / 4).max(1) as u32
729    }
730}
731
732#[async_trait]
733impl ChatDriver for LlmSimDriver {
734    fn effective_context_window(&self, _model: &str) -> Option<usize> {
735        self.config.context_window
736    }
737
738    async fn chat_completion_stream(
739        &self,
740        _endpoint: &everruns_contracts::runtime_provider::ProviderEndpoint,
741        messages: Vec<Message>,
742        config: &LlmCallConfig,
743    ) -> Result<LlmResponseStream> {
744        // Record per-call effort (EVE-595) before any error, so error turns show.
745        if let Some(capture) = &self.config.effort_capture
746            && let Ok(mut efforts) = capture.lock()
747        {
748            efforts.push(config.reasoning_effort.map(|e| e.as_str().to_string()));
749        }
750
751        // Record the provider-visible messages for tests, also before any error.
752        if let Some(capture) = &self.config.message_capture
753            && let Ok(mut calls) = capture.lock()
754        {
755            calls.push(messages.clone());
756        }
757
758        // Check for error configs first
759        if let ResponseConfig::Error(error_msg) = &self.config.response {
760            return Err(anyhow::anyhow!("LLM error: {}", error_msg).into());
761        }
762        if matches!(self.config.response, ResponseConfig::ModelNotAvailable) {
763            return Err(AgentLoopError::model_not_available(config.model.clone()));
764        }
765
766        // Response delay (config or "-ttft-{ms}" in the model name) simulates time to
767        // first token, giving cancellation tests a predictable window.
768        let delay = self
769            .config
770            .response_delay
771            .or_else(|| model_names::ttft(&config.model));
772        if let Some(delay) = delay {
773            tokio::time::sleep(delay).await;
774        }
775
776        // A scenario marker in the user message (or an `llmsim-scenario-*`
777        // model) takes over the whole call, pacing included.
778        if let Some(played) = scenarios::play(
779            &messages,
780            &config.tools,
781            &config.model,
782            self.resolve_latency_profile(&config.model),
783            &self.scenario_attempts,
784        ) {
785            return played;
786        }
787
788        let counters = if let Some(session) = config.metadata.get("session_id") {
789            let mut sessions = self
790                .session_counters
791                .lock()
792                .map_err(|_| AgentLoopError::config("simulator session counters unavailable"))?;
793            if sessions.len() >= 100_000 && !sessions.contains_key(session) {
794                return Err(AgentLoopError::config("simulator session capacity reached"));
795            }
796            sessions
797                .entry(session.clone())
798                .or_insert_with(|| (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))))
799                .clone()
800        } else {
801            (
802                self.response_counter.clone(),
803                self.tool_call_counter.clone(),
804            )
805        };
806        let generated_turn = model_names::realistic_turn(self, &messages, config).map_or_else(
807            || self.generate_turn(&messages, &counters.0, &counters.1),
808            Ok,
809        )?;
810        if generated_turn.stream_stall {
811            return Ok(Box::pin(futures::stream::pending()));
812        }
813        let response_text = generated_turn.text;
814        let tool_calls = generated_turn.tool_calls;
815        let model_name = config.model.clone();
816        let response_id_for_done = self.config.response_id.clone();
817        let latency_profile = self.resolve_latency_profile(&model_name);
818
819        // Calculate token estimates
820        let prompt_tokens: u32 = messages
821            .iter()
822            .map(|m| Self::estimate_tokens(&m.content_as_text()))
823            .sum();
824        let completion_tokens = Self::estimate_tokens(&response_text);
825
826        // TokenStreamBuilder handles TTFT and inter-token delays via LatencyProfile.
827        let usage = Usage {
828            prompt_tokens,
829            completion_tokens,
830            total_tokens: prompt_tokens + completion_tokens,
831            completion_tokens_details: None,
832            prompt_tokens_details: None,
833        };
834
835        let chunk_stream = TokenStreamBuilder::new(&model_name, &response_text)
836            .latency(latency_profile)
837            .usage(usage)
838            .build()
839            .into_chunk_stream();
840
841        // Reasoning artifacts, when the caller asked the model to reason.
842        //
843        // llmsim exists so the agent loop can be exercised without provider
844        // credits, and reasoning was the one part of a turn it could not
845        // produce. That left the whole reasoning path — persistence, the REST
846        // projection, and the sanitization that keeps replay state out of the
847        // API — testable only against live providers, so it went untested.
848        //
849        // The artifact deliberately carries both readable text and opaque
850        // replay state (signature and encrypted payload, as Anthropic and
851        // OpenAI respectively return), so a test can assert that the text
852        // survives to the API while the replay state does not.
853        let reasoning_events: Vec<Result<LlmStreamEvent>> = if config
854            .reasoning_effort
855            .is_some_and(|effort| effort.requests_reasoning())
856        {
857            let part = everruns_contracts::reasoning::ReasoningContentPart::opaque("llmsim")
858                .with_item_id("rs_llmsim_0")
859                .with_signature("llmsim-signature-opaque")
860                .with_encrypted("llmsim-encrypted-opaque")
861                .with_text(everruns_contracts::reasoning::ReasoningText::Plain {
862                    text: LLMSIM_REASONING_TEXT.to_string(),
863                })
864                .with_tokens(8);
865            vec![
866                Ok(LlmStreamEvent::ReasoningDelta {
867                    delta: LLMSIM_REASONING_TEXT.to_string(),
868                    summary: false,
869                }),
870                Ok(LlmStreamEvent::ReasoningItem(part)),
871            ]
872        } else {
873            Vec::new()
874        };
875
876        // Map simulated chunks to stream events, then append tool calls and metadata.
877        let tool_calls_tail = tool_calls;
878        let model_name_done = model_name.clone();
879        let event_stream = chunk_stream.flat_map(move |chunk| {
880            let mut events: Vec<Result<LlmStreamEvent>> = Vec::new();
881
882            for choice in &chunk.choices {
883                if let Some(content) = &choice.delta.content
884                    && !content.is_empty()
885                {
886                    events.push(Ok(LlmStreamEvent::TextDelta(content.clone())));
887                }
888            }
889
890            stream::iter(events)
891        });
892
893        // Append tool calls + done after the text stream
894        let done_events: Vec<Result<LlmStreamEvent>> = {
895            let mut tail = Vec::new();
896            if let Some(calls) = tool_calls_tail {
897                tail.push(Ok(LlmStreamEvent::ToolCalls(calls)));
898            }
899            tail.push(Ok(LlmStreamEvent::Done(Box::new({
900                let mut metadata = LlmCompletionMetadata::default();
901                metadata.total_tokens = Some(prompt_tokens + completion_tokens);
902                metadata.prompt_tokens = Some(prompt_tokens);
903                metadata.completion_tokens = Some(completion_tokens);
904                metadata.model = Some(model_name_done);
905                metadata.response_model = Some(self.config.model_name.clone());
906                metadata.finish_reason = Some("stop".to_string());
907                metadata.response_id = response_id_for_done;
908                metadata
909            }))));
910            tail
911        };
912
913        // Reasoning precedes the answer, matching every real provider.
914        let full_stream = stream::iter(reasoning_events)
915            .chain(event_stream)
916            .chain(stream::iter(done_events));
917        Ok(Box::pin(full_stream))
918    }
919
920    /// Scripted answers stand in for a schema-enforcing provider.
921    fn supports_response_format(&self, _model: &str) -> bool {
922        true
923    }
924}
925
926impl std::fmt::Debug for LlmSimDriver {
927    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
928        f.debug_struct("LlmSimDriver")
929            .field("model", &self.config.model_name)
930            .field("simulate_latency", &self.config.simulate_latency)
931            .finish()
932    }
933}
934
935/// Readable reasoning text llmsim emits when a turn requests reasoning.
936///
937/// Distinctive so a test can assert it reached the API, and assert that the
938/// opaque replay state alongside it did not.
939pub const LLMSIM_REASONING_TEXT: &str = "llmsim deliberating about the request";
940
941// ============================================================================
942// Driver Registration
943// ============================================================================
944
945/// Curated model recommendations for this provider. Refresh when the vendor
946/// rotates flagships; keep every role default inside `recommended_models`.
947pub const LLMSIM_RECOMMENDED: everruns_contracts::recommended_models::ProviderRecommendedModels =
948    everruns_contracts::recommended_models::ProviderRecommendedModels {
949        provider: "llmsim",
950        recommended_models: &["llmsim-default"],
951        coding_model: "llmsim-default",
952        research_model: "llmsim-default",
953        chat_model: "llmsim-default",
954        small_model: "llmsim-default",
955        decision_model: "llmsim-default",
956    };
957
958/// Register the LlmSim driver with the driver registry
959///
960/// This registers a driver for the `LlmSim` provider type.
961/// The driver is created with a default configuration; for custom configs,
962/// create the driver directly using `LlmSimDriver::new()`.
963///
964/// # Example
965///
966/// ```ignore
967/// use everruns_core::DriverRegistry;
968/// use everruns_llmsim::register_driver;
969///
970/// let mut registry = DriverRegistry::new();
971/// register_driver(&mut registry);
972/// ```
973pub fn register_driver(registry: &mut DriverRegistry) {
974    let mut descriptor = DriverDescriptor::chat_only(DriverId::LlmSim, |_config| {
975        // Default driver - tests can create custom drivers directly.
976        Box::new(LlmSimDriver::default_driver()) as BoxedChatDriver
977    });
978    descriptor.recommended_models = Some(LLMSIM_RECOMMENDED);
979    registry.register_descriptor_or_replace(realtime::with_simulated_realtime(descriptor));
980}
981
982/// Register the LlmSim driver with a custom configuration. Useful for
983/// servers/workers that want to opt into a scripted scenario (e.g. the
984/// `user_hooks` audit-log demo) without changing the default behaviour for
985/// callers that don't.
986///
987/// The same `config` is cloned for every constructed driver in the
988/// registry, so its `Arc`-backed counters (sequence index, etc.) are
989/// shared across invocations.
990pub fn register_driver_with_config(registry: &mut DriverRegistry, config: LlmSimConfig) {
991    let driver = LlmSimDriver::new(config);
992    let mut descriptor = DriverDescriptor::chat_only(DriverId::LlmSim, move |_config| {
993        Box::new(driver.clone()) as BoxedChatDriver
994    });
995    descriptor.recommended_models = Some(LLMSIM_RECOMMENDED);
996    registry.register_descriptor_or_replace(realtime::with_simulated_realtime(descriptor));
997}
998
999/// Create a LlmSim driver with custom configuration
1000///
1001/// This is the preferred way to create a driver in tests.
1002/// Unlike `register_driver`, this gives you full control over the config.
1003///
1004/// # Example
1005///
1006/// ```ignore
1007/// use everruns_llmsim::{create_chat_driver, LlmSimConfig};
1008///
1009/// let driver = create_chat_driver(
1010///     LlmSimConfig::fixed("I'll help you with that!")
1011///         .with_tool_calls(vec![...])
1012/// );
1013/// ```
1014pub fn create_chat_driver(config: LlmSimConfig) -> BoxedChatDriver {
1015    Box::new(LlmSimDriver::new(config))
1016}
1017
1018// ============================================================================
1019// Tests
1020// ============================================================================
1021
1022#[cfg(test)]
1023mod tests;