Skip to main content

roder_core/
runtime.rs

1use std::collections::{HashMap, HashSet};
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12    model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33    ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37    RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42    TeamMemberStatus,
43};
44use roder_api::thread::{
45    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51    TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67    classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73    route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78    apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83    provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
94/// limit into a continuation, or as the empty-tool-call persistence nudge, so
95/// the model keeps working instead of ending the turn early.
96const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
98/// its render loop, which during an active turn only wakes at the ~6 FPS status
99/// animation cadence (backend events do not wake the input poll), so up to
100/// ~166ms of events buffer between drains. A reasoning-high provider that runs
101/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
102/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
103/// 1024-slot buffer overflowed in those windows and silently dropped tool and
104/// thinking rows from the live view. Sized for generous headroom across bursts
105/// and brief render stalls.
106const EVENT_BUS_CAPACITY: usize = 16_384;
107
108/// Turn id reported to session-scoped local hooks. `SessionStart` and
109/// `SessionEnd` bracket the session rather than any one turn, but the hook
110/// payload carries a turn id, so they share this synthetic value.
111const SESSION_HOOK_TURN_ID: &str = "session";
112pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
113pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
114const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
115const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
116const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
117pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
118const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
119const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
120const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
121
122#[derive(Clone, Copy, Debug, Eq, PartialEq)]
123enum InferenceTimeoutAction {
124    ScoreableCheckpoint,
125    Finalization,
126}
127
128#[derive(Debug, Clone)]
129pub struct RuntimeConfig {
130    pub default_provider: String,
131    pub default_model: String,
132    pub reasoning: Option<String>,
133    pub auto_compact_token_limit: Option<u32>,
134    pub file_backed_dynamic_context: bool,
135    pub hosted_web_search: HostedWebSearchConfig,
136    pub tool_search: ToolSearchConfig,
137    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
138    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
139    pub model_edit_tools: HashMap<String, String>,
140    pub model_parallel_tool_calls: HashMap<String, bool>,
141    pub model_profiles: HashMap<String, ModelHarnessProfile>,
142    pub tool_allowlist: Vec<String>,
143    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
144    pub external_tool_timeout_seconds: u64,
145    pub command_shell: String,
146    pub workspace: Option<String>,
147    pub policy_mode: PolicyMode,
148    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
149    /// injects the swarm reminder into each turn's developer instructions so any
150    /// client benefits, not just the TUI.
151    pub agent_swarm_mode: bool,
152    /// Whether ultra mode is active. When on, the runtime injects proactive
153    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
154    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
155    /// proactive multi-agent for that model without requiring this flag.
156    pub ultra_mode: bool,
157    pub runtime_profile: RuntimeProfile,
158    pub inference_router: RuntimeInferenceRouterConfig,
159    pub speed_policy: RuntimeSpeedPolicyConfig,
160    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
161    pub reliability: RuntimeReliabilityConfig,
162    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
163    pub turn_deadline_seconds: Option<u64>,
164    pub remote_runner_destination: Option<RunnerDestination>,
165    pub team_data_dir: Option<PathBuf>,
166    pub roadmap_data_dir: Option<PathBuf>,
167    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
168    /// `[review]` settings: the review sub-turn and its publishers.
169    pub review: crate::review::RuntimeReviewConfig,
170}
171
172impl Default for RuntimeConfig {
173    fn default() -> Self {
174        Self {
175            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
176            default_model: "mock".to_string(),
177            reasoning: None,
178            auto_compact_token_limit: None,
179            file_backed_dynamic_context: true,
180            hosted_web_search: HostedWebSearchConfig::cached(),
181            tool_search: ToolSearchConfig::default(),
182            provider_tool_search: HashMap::new(),
183            model_tool_search: HashMap::new(),
184            model_edit_tools: HashMap::new(),
185            model_parallel_tool_calls: HashMap::new(),
186            model_profiles: HashMap::new(),
187            tool_allowlist: Vec::new(),
188            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
189            command_shell: roder_api::command_shell::default_command_shell(),
190            workspace: None,
191            policy_mode: PolicyMode::Default,
192            agent_swarm_mode: false,
193            ultra_mode: false,
194            runtime_profile: RuntimeProfile::Interactive,
195            inference_router: RuntimeInferenceRouterConfig::default(),
196            speed_policy: RuntimeSpeedPolicyConfig::default(),
197            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
198            reliability: RuntimeReliabilityConfig::default(),
199            turn_deadline_seconds: None,
200            remote_runner_destination: None,
201            team_data_dir: None,
202            roadmap_data_dir: None,
203            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
204            review: crate::review::RuntimeReviewConfig::default(),
205        }
206    }
207}
208
209#[derive(Debug, Clone)]
210pub struct StartTurnRequest {
211    pub thread_id: ThreadId,
212    pub message: String,
213    pub images: Vec<InputImage>,
214    pub provider_override: Option<String>,
215    pub model_override: Option<String>,
216    pub reasoning_override: Option<String>,
217    pub workspace: String,
218    pub instructions: InstructionBundle,
219    /**
220     * Per-turn developer-authority context for this turn's InstructionBundle.
221     * Applies to every inference round of the turn, is never written to
222     * thread state, and does not carry over to later turns.
223     */
224    pub developer_context: Option<String>,
225    pub task_ledger_required: bool,
226}
227
228#[derive(Debug, Clone)]
229pub struct CreateThreadRequest {
230    pub title: Option<String>,
231    pub workspace: String,
232    pub workspace_id: Option<String>,
233    pub root_id: Option<String>,
234    pub provider: Option<String>,
235    pub model: Option<String>,
236    pub selection_mode: Option<ModelSelectionMode>,
237    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
238    pub tool_allowlist: Vec<String>,
239    /// Host-supplied instructions added to the developer slot of every turn's inference request.
240    pub developer_instructions: Option<String>,
241    /// Host-executed tool specs advertised to the model on every turn of this thread.
242    pub external_tools: Vec<roder_api::tools::ToolSpec>,
243    /// Explicit remote-runner binding for the thread's native coding tools.
244    pub runner: Option<ThreadRunnerSelection>,
245}
246
247/**
248 * Thread-level remote-runner selection. The destination config is persisted
249 * with the thread, so secrets must reach the provider through its
250 * environment, not this config.
251 */
252#[derive(Debug, Clone)]
253pub struct ThreadRunnerSelection {
254    pub provider_id: String,
255    pub config: serde_json::Value,
256    /// Absolute path on the runner used as the thread's coding-tool workspace root.
257    pub workspace: String,
258    /**
259     * Extra absolute runner paths file reads may resolve under, beyond
260     * `workspace`. Writes and the working directory stay confined to
261     * `workspace`.
262     */
263    pub read_roots: Vec<String>,
264}
265
266#[derive(Debug, Clone, PartialEq, Eq)]
267pub struct PendingPlanExit {
268    pub thread_id: ThreadId,
269    pub turn_id: TurnId,
270    pub request_id: String,
271    pub target_mode: PolicyMode,
272    pub plan_summary: Option<String>,
273    pub next_steps: Vec<String>,
274    pub requested_at: OffsetDateTime,
275    pub expires_at: Option<OffsetDateTime>,
276}
277
278pub(crate) struct PendingToolApproval {
279    pub(crate) thread_id: ThreadId,
280    pub(crate) turn_id: TurnId,
281    pub(crate) tool_id: String,
282    pub(crate) tool_name: String,
283    pub(crate) call: roder_api::tools::ToolCall,
284    pub(crate) tx: oneshot::Sender<bool>,
285}
286
287pub(crate) struct PendingUserInput {
288    pub(crate) thread_id: ThreadId,
289    pub(crate) turn_id: TurnId,
290    pub(crate) tx: oneshot::Sender<serde_json::Value>,
291}
292
293/// Host answer to an external tool call delivered via `tools/resolve`.
294#[derive(Debug, Clone, PartialEq, Eq)]
295pub struct ExternalToolResolution {
296    pub output: String,
297    pub is_error: bool,
298}
299
300pub(crate) struct PendingExternalToolCall {
301    pub(crate) thread_id: ThreadId,
302    pub(crate) turn_id: TurnId,
303    pub(crate) tool_id: String,
304    pub(crate) tool_name: String,
305    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
306}
307
308#[derive(Clone)]
309struct ActiveTurnHandle {
310    thread_id: ThreadId,
311    abort: AbortHandle,
312    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
313    drain: Arc<TurnDrainHandle>,
314}
315
316/// Tracks a task after it has been interrupted and removed from the interactive
317/// active-turn map. This lets users start a follow-up turn immediately while a
318/// bounded runtime shutdown can still await the original task's cleanup.
319struct TurnDrainHandle {
320    thread_id: ThreadId,
321    interrupt_requested: AtomicBool,
322    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
323    completed: AtomicBool,
324    completed_notify: Notify,
325}
326
327/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
328/// tasks have reached their cleanup paths; provider-specific child-process
329/// reaping remains an explicit provider capability.
330#[derive(Debug, Clone, PartialEq, Eq)]
331pub enum RuntimeDrainOutcome {
332    Clean {
333        interrupted_turn_ids: Vec<TurnId>,
334    },
335    DeadlineExceeded {
336        interrupted_turn_ids: Vec<TurnId>,
337        remaining_turn_ids: Vec<TurnId>,
338    },
339    /// At least one lifecycle transition initiated by this drain could not be
340    /// durably appended. The runtime still made its best cleanup attempt, but
341    /// callers must not claim a persisted terminal state as proof of recovery.
342    PersistenceFailed {
343        interrupted_turn_ids: Vec<TurnId>,
344        remaining_turn_ids: Vec<TurnId>,
345    },
346}
347
348#[derive(Clone)]
349struct QueuedTurnSteer {
350    message: UserMessage,
351    mailbox_ack: Option<MailboxDeliveryAck>,
352}
353
354#[derive(Clone)]
355struct MailboxDeliveryAck {
356    team_id: TeamId,
357    message_ids: Vec<String>,
358}
359
360#[derive(Clone)]
361struct InheritedTurnContext {
362    workspace: String,
363    instructions: InstructionBundle,
364    developer_context: Option<String>,
365}
366
367#[derive(Debug, Clone, Default, PartialEq, Eq)]
368pub struct ThreadActivity {
369    pub active_turn_id: Option<TurnId>,
370    pub active_flags: Vec<String>,
371}
372
373/// Per-thread settings persisted at thread creation and applied to every turn.
374#[derive(Debug, Clone, Default)]
375pub(crate) struct ThreadTurnOverrides {
376    pub(crate) tool_allowlist: Vec<String>,
377    pub(crate) developer_instructions: Option<String>,
378    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
379}
380
381#[derive(Debug, Clone, Copy, PartialEq, Eq)]
382pub(crate) enum TurnRunOutcome {
383    Completed,
384    Stopped,
385}
386
387impl PendingPlanExit {
388    pub fn new(
389        thread_id: ThreadId,
390        turn_id: TurnId,
391        request_id: String,
392        target_mode: PolicyMode,
393        plan_summary: Option<String>,
394        next_steps: Vec<String>,
395    ) -> Self {
396        let requested_at = OffsetDateTime::now_utc();
397        Self {
398            thread_id,
399            turn_id,
400            request_id,
401            target_mode,
402            plan_summary,
403            next_steps,
404            requested_at,
405            expires_at: Some(requested_at + default_plan_exit_timeout()),
406        }
407    }
408
409    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
410        self.expires_at.is_some_and(|expires_at| now >= expires_at)
411    }
412}
413
414pub fn default_plan_exit_timeout() -> Duration {
415    Duration::minutes(10)
416}
417
418pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
419
420fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
421    messages
422        .iter()
423        .map(|message| {
424            let recipient = team
425                .members
426                .iter()
427                .find(|member| member.id == message.to_member_id)
428                .map(canonical_team_member_path)
429                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
430            let sender = message
431                .from_member_id
432                .as_deref()
433                .and_then(|id| team.members.iter().find(|member| member.id == id))
434                .map(canonical_team_member_path)
435                .unwrap_or_else(|| "/root".to_string());
436            let message_type = match message.kind {
437                TeamMailboxMessageKind::Message => "MESSAGE",
438                TeamMailboxMessageKind::NewTask => "NEW_TASK",
439                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
440            };
441            format!(
442                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
443                message.text
444            )
445        })
446        .collect::<Vec<_>>()
447        .join("\n\n")
448}
449
450fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
451    if let Some(agent_path) = member
452        .agent_path
453        .as_deref()
454        .filter(|path| *path == "/root" || path.starts_with("/root/"))
455    {
456        return agent_path.to_string();
457    }
458    if member.role == TeamMemberRole::Lead {
459        return "/root".to_string();
460    }
461    member
462        .task_name
463        .as_deref()
464        .filter(|name| !name.trim().is_empty())
465        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
466        .unwrap_or_else(|| format!("/root/{}", member.id))
467}
468
469fn is_codex_v2_team(team: &TeamState) -> bool {
470    team.members.iter().any(|member| {
471        member
472            .model
473            .as_deref()
474            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
475    })
476}
477
478fn runtime_local_team_view(
479    mut team: TeamState,
480    active_thread_ids: &std::collections::HashSet<ThreadId>,
481) -> TeamState {
482    for member in &mut team.members {
483        if member.status == TeamMemberStatus::Running
484            && !active_thread_ids.contains(&member.thread_id)
485        {
486            member.status = TeamMemberStatus::Interrupted;
487            member.current_turn_id = None;
488        }
489    }
490    team
491}
492
493type RunnerToolExecutionKey = (String, String);
494type RunnerToolExecutionMutex = Mutex<()>;
495type RunnerToolExecutionLockRegistry =
496    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
497
498pub struct Runtime {
499    pub bus: EventBus,
500    pub registry: ExtensionRegistry,
501    config: RwLock<RuntimeConfig>,
502    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
503    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
504    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
505    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
506    /// Serializes turn admission with shutdown quiescing. A drain takes this
507    /// gate before flipping `accepting_turns` and snapshotting active work so a
508    /// turn that observed the old flag cannot be inserted after the snapshot.
509    turn_admission: Mutex<()>,
510    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
511    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
512    /// Provider cleanup handles register synchronously through the inference
513    /// tool-executor context. They are intentionally separate from active turn
514    /// admission so an interrupted turn can remain drainable after a follow-up
515    /// turn starts on the same thread.
516    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
517    accepting_turns: AtomicBool,
518    /// Monotonic counter used by bounded drains to surface lifecycle-state
519    /// persistence failures without making an individual provider turn fail.
520    lifecycle_persistence_failures: AtomicUsize,
521    lifecycle_shutdown_drains: AtomicUsize,
522    lifecycle_clean_shutdowns: AtomicUsize,
523    lifecycle_deadline_exceeded: AtomicUsize,
524    lifecycle_persistence_failed_drains: AtomicUsize,
525    lifecycle_restart_reconciliations: AtomicUsize,
526    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
527    provider_cleanup_confirmed: AtomicUsize,
528    provider_cleanup_timed_out: AtomicUsize,
529    provider_cleanup_unknown: AtomicUsize,
530    active_turns_changed: Notify,
531    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
532    /// Threads that have already dispatched a `SessionStart` local hook.
533    ///
534    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
535    /// thread from its store, so without this a session ran its setup hooks
536    /// several times — including after the turn had already stopped.
537    session_hook_started: RwLock<HashSet<ThreadId>>,
538    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
539    /// Process-local execution boundary. Local/CLI runtimes default to true;
540    /// hosted runtime pools set this false so missing runner metadata fails
541    /// closed instead of exposing the host workspace.
542    allow_local_workspaces: AtomicBool,
543    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
544    // This stays process-local because developer_context is explicitly volatile and must never
545    // be persisted to thread state.
546    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
547    workspace: PathBuf,
548    pub(crate) teams: TeamManager,
549    agent_team_spawn_lock: Mutex<()>,
550    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
551    /// Completed reviews, most recent last, so a later publish can resolve a
552    /// review id back to its findings. Bounded; see `review::run`.
553    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
554    pub(crate) goals: Arc<RuntimeGoalController>,
555    context_artifacts: roder_api::artifacts::ContextArtifactStore,
556    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
557    thread_item_cache: Mutex<ThreadItemCache>,
558    pub(crate) tool_registry: ToolRegistry,
559    media_generation: Arc<crate::media_generation::MediaGenerationService>,
560    pub(crate) skills: RwLock<SkillRegistry>,
561    /// Lazily-started bounded dispatch of emitted events to registry
562    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
563    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
564    pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
565    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
566    /// this map uses its stored value; absent threads fall back to the
567    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
568    /// team per-member policy-mode override idiom so swarm mode is per-thread
569    /// like the other `thread/*` operations, without a separate runtime.
570    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
571    /// Per-thread ultra mode overrides. A thread present in this map uses its
572    /// stored value; absent threads fall back to the runtime-global
573    /// `RuntimeConfig.ultra_mode` default.
574    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
575    /**
576     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
577     * held across provider initialization, making the first workspace-tool
578     * access a singleflight while allowing unrelated threads to initialize in
579     * parallel. Failed initialization is shared with current waiters, then
580     * evicted so a later tool call can retry.
581     */
582    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
583    /**
584     * Outer tool-call locks keyed by the actual remote session, not the
585     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
586     * serializes threads that share one sandbox without coupling different
587     * tenants or independent runner sessions.
588     */
589    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
590}
591
592#[derive(Clone)]
593struct CachedRunnerSession {
594    destination: RunnerDestination,
595    session: Arc<dyn RemoteRunnerSession>,
596}
597
598struct RunnerSessionSlot {
599    provider_id: String,
600    destination_id: String,
601    state: Mutex<RunnerSessionSlotState>,
602    initialized: Notify,
603}
604
605enum RunnerSessionSlotState {
606    Empty,
607    Initializing,
608    Ready(CachedRunnerSession),
609    Failed(Arc<str>),
610}
611
612impl RunnerSessionSlot {
613    fn empty(destination: &RunnerDestination) -> Self {
614        Self {
615            provider_id: destination.provider_id.clone(),
616            destination_id: destination.id.clone(),
617            state: Mutex::new(RunnerSessionSlotState::Empty),
618            initialized: Notify::new(),
619        }
620    }
621
622    fn ready(session: CachedRunnerSession) -> Self {
623        Self {
624            provider_id: session.destination.provider_id.clone(),
625            destination_id: session.destination.id.clone(),
626            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
627            initialized: Notify::new(),
628        }
629    }
630
631    fn matches(&self, destination: &RunnerDestination) -> bool {
632        self.provider_id == destination.provider_id && self.destination_id == destination.id
633    }
634}
635
636impl Runtime {
637    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
638        if registry.inference_engines.is_empty() {
639            anyhow::bail!("at least one inference engine must be registered");
640        }
641        validate_runtime_config_reasoning(&config)?;
642        validate_runtime_inference_router_config(&registry, &config)?;
643
644        let bus = EventBus::new(EVENT_BUS_CAPACITY);
645        let thread_store = registry
646            .thread_stores
647            .first()
648            .map(|factory| factory.create());
649        let mut tool_registry = ToolRegistry::default();
650        for contributor in &registry.tools {
651            contributor
652                .contribute(&mut tool_registry)
653                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
654        }
655        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
656
657        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
658            registry.media_generator_providers.clone(),
659            config.media_generation.clone(),
660        ));
661        tool_registry.replace(Arc::new(
662            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
663        ));
664
665        let team_data_dir = config.team_data_dir.clone();
666        let workspace = config
667            .workspace
668            .clone()
669            .map(PathBuf::from)
670            .unwrap_or(std::env::current_dir()?);
671        let roadmap_data_dir = config
672            .roadmap_data_dir
673            .clone()
674            .unwrap_or_else(|| workspace.join(".roder"));
675        let context_artifacts = thread_store
676            .as_ref()
677            .and_then(|store| store.context_artifact_store())
678            .or_else(|| {
679                thread_store
680                    .as_ref()
681                    .and_then(|store| store.local_thread_root())
682                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
683            })
684            .unwrap_or_else(|| {
685                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
686            });
687        let goals = Arc::new(RuntimeGoalController::new(
688            bus.clone(),
689            thread_store.clone(),
690        ));
691        let runtime = Self {
692            bus,
693            registry,
694            config: RwLock::new(config),
695            pending_plan_exit: RwLock::new(None),
696            pending_tool_approvals: Mutex::new(HashMap::new()),
697            pending_user_inputs: Mutex::new(HashMap::new()),
698            pending_external_tool_calls: Mutex::new(HashMap::new()),
699            turn_admission: Mutex::new(()),
700            active_turns: RwLock::new(HashMap::new()),
701            turn_drains: RwLock::new(HashMap::new()),
702            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
703            accepting_turns: AtomicBool::new(true),
704            lifecycle_persistence_failures: AtomicUsize::new(0),
705            lifecycle_shutdown_drains: AtomicUsize::new(0),
706            lifecycle_clean_shutdowns: AtomicUsize::new(0),
707            lifecycle_deadline_exceeded: AtomicUsize::new(0),
708            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
709            lifecycle_restart_reconciliations: AtomicUsize::new(0),
710            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
711            provider_cleanup_confirmed: AtomicUsize::new(0),
712            provider_cleanup_timed_out: AtomicUsize::new(0),
713            provider_cleanup_unknown: AtomicUsize::new(0),
714            active_turns_changed: Notify::new(),
715            active_turn_selections: RwLock::new(HashMap::new()),
716            session_hook_started: RwLock::new(HashSet::new()),
717            active_turn_contexts: RwLock::new(HashMap::new()),
718            allow_local_workspaces: AtomicBool::new(true),
719            team_member_turn_contexts: Mutex::new(HashMap::new()),
720            workspace: workspace.clone(),
721            teams: TeamManager::new(
722                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
723            ),
724            agent_team_spawn_lock: Mutex::new(()),
725            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
726                workspace,
727                roadmap_data_dir,
728            )),
729            reviews: Mutex::new(Vec::new()),
730            goals,
731            context_artifacts,
732            thread_store,
733            thread_item_cache: Mutex::new(ThreadItemCache::default()),
734            tool_registry,
735            media_generation,
736            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
737                PathBuf::new(),
738            ))),
739            event_sink_dispatcher: tokio::sync::OnceCell::new(),
740            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
741            agent_swarm_modes: RwLock::new(HashMap::new()),
742            ultra_modes: RwLock::new(HashMap::new()),
743            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
744            runner_tool_execution_locks: Mutex::new(HashMap::new()),
745        };
746        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
747            timestamp: OffsetDateTime::now_utc(),
748        }));
749        for manifest in &runtime.registry.manifests {
750            runtime
751                .bus
752                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
753                    extension_id: manifest.id.clone(),
754                    timestamp: OffsetDateTime::now_utc(),
755                }));
756        }
757        Ok(runtime)
758    }
759
760    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
761        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
762        builder.inference_engine(engine);
763        Self::new(builder.build()?, RuntimeConfig::default())
764    }
765
766    pub fn fake() -> anyhow::Result<Self> {
767        Self::from_engine(Arc::new(FakeInferenceEngine))
768    }
769
770    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
771        self.bus.subscribe()
772    }
773
774    /// Returns process-local, redacted lifecycle health counters. The snapshot
775    /// intentionally has fixed fields and contains no provider, command,
776    /// process, thread, or turn identifiers.
777    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
778        LifecycleMetricsSnapshot {
779            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
780            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
781            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
782                as u64,
783            persistence_failed_count: self
784                .lifecycle_persistence_failed_drains
785                .load(Ordering::Acquire) as u64,
786            restart_reconciliation_count: self
787                .lifecycle_restart_reconciliations
788                .load(Ordering::Acquire) as u64,
789            lifecycle_persistence_failure_count: self
790                .lifecycle_persistence_failures
791                .load(Ordering::Acquire) as u64,
792            shutdown_drain_duration_ms_total: self
793                .lifecycle_shutdown_drain_duration_ms_total
794                .load(Ordering::Acquire) as u64,
795            provider_cleanup_confirmed_count: self
796                .provider_cleanup_confirmed
797                .load(Ordering::Acquire) as u64,
798            provider_cleanup_timed_out_count: self
799                .provider_cleanup_timed_out
800                .load(Ordering::Acquire) as u64,
801            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
802                as u64,
803        }
804    }
805
806    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
807        let Some(store) = &self.thread_store else {
808            return true;
809        };
810        let state = match record.extension_state() {
811            Ok(state) => state,
812            Err(_) => {
813                self.lifecycle_persistence_failures
814                    .fetch_add(1, Ordering::AcqRel);
815                return false;
816            }
817        };
818        // Lifecycle diagnostics must not turn a provider result into a failed
819        // turn merely because a third-party store lacks extension-state support.
820        // JSONL/Postgres stores persist this append-only record atomically at
821        // their own storage boundary.
822        if store
823            .append_extension_state(&record.thread_id, &state)
824            .await
825            .is_err()
826        {
827            self.lifecycle_persistence_failures
828                .fetch_add(1, Ordering::AcqRel);
829            return false;
830        }
831        true
832    }
833
834    async fn record_turn_lifecycle(
835        &self,
836        thread_id: ThreadId,
837        turn_id: TurnId,
838        state: TurnLifecycleState,
839        cleanup: TurnCleanupState,
840        reason: Option<TurnLifecycleReason>,
841    ) -> TurnLifecycleRecord {
842        self.record_turn_lifecycle_with_ownership(
843            thread_id,
844            turn_id,
845            state,
846            cleanup,
847            reason,
848            TurnCleanupOwnership::RuntimeTaskOnly,
849        )
850        .await
851    }
852
853    async fn record_turn_lifecycle_with_ownership(
854        &self,
855        thread_id: ThreadId,
856        turn_id: TurnId,
857        state: TurnLifecycleState,
858        cleanup: TurnCleanupState,
859        reason: Option<TurnLifecycleReason>,
860        ownership: TurnCleanupOwnership,
861    ) -> TurnLifecycleRecord {
862        let record = TurnLifecycleRecord::new(
863            thread_id,
864            turn_id,
865            state,
866            cleanup,
867            reason,
868            OffsetDateTime::now_utc(),
869        )
870        .with_ownership(ownership);
871        let _ = self.persist_turn_lifecycle_record(&record).await;
872        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
873            .await;
874        record
875    }
876
877    pub(crate) fn register_provider_turn_cleanup(
878        &self,
879        turn_id: &TurnId,
880        cleanup: Arc<dyn ProviderTurnCleanup>,
881    ) {
882        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
883            cleanups.insert(turn_id.clone(), cleanup);
884        }
885    }
886
887    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
888        self.provider_turn_cleanups
889            .lock()
890            .ok()
891            .and_then(|cleanups| cleanups.get(turn_id).cloned())
892            .map(|cleanup| cleanup.ownership())
893            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
894    }
895
896    async fn await_provider_turn_cleanup(
897        &self,
898        turn_id: &TurnId,
899    ) -> (TurnCleanupState, TurnCleanupOwnership) {
900        let cleanup = self
901            .provider_turn_cleanups
902            .lock()
903            .ok()
904            .and_then(|mut cleanups| cleanups.remove(turn_id));
905        let Some(cleanup) = cleanup else {
906            return (
907                TurnCleanupState::Unknown,
908                TurnCleanupOwnership::RuntimeTaskOnly,
909            );
910        };
911        let ownership = cleanup.ownership();
912        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
913            Ok(Ok(())) => {
914                self.provider_cleanup_confirmed
915                    .fetch_add(1, Ordering::AcqRel);
916                (
917                    TurnCleanupState::Completed,
918                    TurnCleanupOwnership::ProviderCleanupConfirmed,
919                )
920            }
921            Ok(Err(_)) => {
922                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
923                (TurnCleanupState::Unknown, ownership)
924            }
925            Err(_) => {
926                self.provider_cleanup_timed_out
927                    .fetch_add(1, Ordering::AcqRel);
928                (TurnCleanupState::TimedOut, ownership)
929            }
930        }
931    }
932
933    fn record_lifecycle_drain_outcome(
934        &self,
935        started_at: tokio::time::Instant,
936        outcome: &RuntimeDrainOutcome,
937    ) {
938        self.lifecycle_shutdown_drains
939            .fetch_add(1, Ordering::AcqRel);
940        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
941            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
942            Ordering::AcqRel,
943        );
944        match outcome {
945            RuntimeDrainOutcome::Clean { .. } => {
946                self.lifecycle_clean_shutdowns
947                    .fetch_add(1, Ordering::AcqRel);
948            }
949            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
950                self.lifecycle_deadline_exceeded
951                    .fetch_add(1, Ordering::AcqRel);
952            }
953            RuntimeDrainOutcome::PersistenceFailed { .. } => {
954                self.lifecycle_persistence_failed_drains
955                    .fetch_add(1, Ordering::AcqRel);
956            }
957        }
958    }
959
960    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
961        match event {
962            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
963                event.thread_id.clone(),
964                event.turn_id.clone(),
965                TurnLifecycleState::Running,
966                TurnCleanupState::NotRequested,
967                None,
968                event.timestamp,
969            )),
970            // `run_turn` persists completed turns after it has awaited a
971            // registered provider cleanup handle. Do not let the generic event
972            // projection overwrite that acknowledgement with an unproven state.
973            RoderEvent::TurnCompleted(_) => None,
974            // Some failure paths already have an event before the turn wrapper
975            // reaches its cleanup acknowledgement. Preserve a durable fallback
976            // reason here; the wrapper appends a later record with the more
977            // precise cleanup outcome when a provider registered one.
978            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
979                event.thread_id.clone(),
980                event.turn_id.clone(),
981                TurnLifecycleState::Failed,
982                TurnCleanupState::Unknown,
983                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
984                    TurnLifecycleReason::DeadlineExceeded
985                } else {
986                    TurnLifecycleReason::ProviderFailure
987                }),
988                event.timestamp,
989            )),
990            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
991            RoderEvent::TurnLifecycleUpdated(_) => None,
992            _ => None,
993        }
994    }
995
996    pub fn registry(&self) -> &ExtensionRegistry {
997        &self.registry
998    }
999
1000    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1001        self.media_generation.clone()
1002    }
1003
1004    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1005        self.context_artifacts.clone()
1006    }
1007
1008    pub async fn execute_workflow_tool(
1009        &self,
1010        thread_id: ThreadId,
1011        tool_name: &str,
1012        arguments: serde_json::Value,
1013    ) -> anyhow::Result<ToolResult> {
1014        let Some(executor) = self.tool_registry.get(tool_name) else {
1015            anyhow::bail!("tool not found: {tool_name}");
1016        };
1017        let tool_call = ToolCall {
1018            id: format!("slash-{tool_name}"),
1019            name: tool_name.to_string(),
1020            raw_arguments: serde_json::to_string(&arguments)?,
1021            arguments,
1022            thread_id: thread_id.clone(),
1023            turn_id: "slash-command".to_string(),
1024        };
1025        let runtime_config = self.status().await;
1026        let ctx = self.tool_execution_context(
1027            thread_id,
1028            "slash-command".to_string(),
1029            runtime_config.policy_mode,
1030            runtime_config.workspace.as_deref(),
1031            Some(&runtime_config.command_shell),
1032        );
1033        executor.execute(ctx, tool_call).await
1034    }
1035
1036    pub(crate) fn tool_execution_context(
1037        &self,
1038        thread_id: ThreadId,
1039        turn_id: TurnId,
1040        mode: PolicyMode,
1041        workspace: Option<&str>,
1042        command_shell: Option<&str>,
1043    ) -> ToolExecutionContext {
1044        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1045            .with_command_shell(command_shell.unwrap_or_default())
1046            .with_process_runner(Arc::new(LocalProcessRunner))
1047            .with_context_artifacts(self.context_artifacts.backend())
1048            .with_goal_controller(self.goals.clone())
1049            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1050                self.bus.clone(),
1051                self.thread_store.clone(),
1052            )))
1053            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1054                self.bus.clone(),
1055                self.thread_store.clone(),
1056            )));
1057        if let Some(workspace) = workspace {
1058            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1059        }
1060        ctx
1061    }
1062
1063    pub async fn status(&self) -> RuntimeConfig {
1064        self.config.read().await.clone()
1065    }
1066
1067    pub async fn set_skills(&self, skills: SkillRegistry) {
1068        *self.skills.write().await = skills;
1069    }
1070
1071    pub async fn skills_snapshot(&self) -> SkillRegistry {
1072        self.skills.read().await.clone()
1073    }
1074
1075    pub fn workspace(&self) -> PathBuf {
1076        self.workspace.clone()
1077    }
1078
1079    /// Controls whether native workspace tools may execute without an
1080    /// explicit remote-runner binding. Hosted runtimes disable this after
1081    /// construction; ordinary local runtimes retain the compatible default.
1082    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1083        self.allow_local_workspaces
1084            .store(allowed, Ordering::Relaxed);
1085    }
1086
1087    pub fn allows_local_workspaces(&self) -> bool {
1088        self.allow_local_workspaces.load(Ordering::Relaxed)
1089    }
1090
1091    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1092        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1093            destination_id: destination.id.clone(),
1094            provider_id: destination.provider_id.clone(),
1095            state: "configured".to_string(),
1096            session_id: None,
1097            timestamp: OffsetDateTime::now_utc(),
1098        });
1099        self.config.write().await.remote_runner_destination = destination;
1100        if let Some(lifecycle) = lifecycle {
1101            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1102        } else {
1103            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1104                destination_id: "local".to_string(),
1105                provider_id: "local".to_string(),
1106                state: "local_fallback".to_string(),
1107                session_id: None,
1108                timestamp: OffsetDateTime::now_utc(),
1109            }))
1110            .await;
1111        }
1112    }
1113
1114    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1115        let mut cfg = self.config.write().await;
1116        cfg.file_backed_dynamic_context = enabled;
1117        cfg.clone()
1118    }
1119
1120    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1121        let mut cfg = self.config.write().await;
1122        cfg.command_shell = shell;
1123        cfg.clone()
1124    }
1125
1126    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1127        let mut pending = self.pending_plan_exit.write().await;
1128        let current = pending.clone()?;
1129        if !current.is_expired(OffsetDateTime::now_utc()) {
1130            return Some(current);
1131        }
1132        *pending = None;
1133        drop(pending);
1134        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1135            .await;
1136        None
1137    }
1138
1139    pub async fn set_policy_mode(
1140        &self,
1141        mode: PolicyMode,
1142        reason: Option<String>,
1143    ) -> anyhow::Result<RuntimeConfig> {
1144        let mut cfg = self.config.write().await;
1145        let previous_mode = cfg.policy_mode;
1146        cfg.policy_mode = mode;
1147        let next = cfg.clone();
1148        drop(cfg);
1149        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1150            thread_id: "runtime".to_string(),
1151            turn_id: None,
1152            previous_mode,
1153            new_mode: mode,
1154            reason,
1155            timestamp: OffsetDateTime::now_utc(),
1156        }))
1157        .await;
1158        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1159            .await;
1160        Ok(next)
1161    }
1162
1163    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1164    /// swarm reminder is injected into each turn's developer instructions so
1165    /// every app-server/SDK client gets it, not only the TUI.
1166    pub async fn set_agent_swarm_mode(
1167        &self,
1168        enabled: bool,
1169        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1170    ) -> anyhow::Result<RuntimeConfig> {
1171        let mut cfg = self.config.write().await;
1172        cfg.agent_swarm_mode = enabled;
1173        let next = cfg.clone();
1174        drop(cfg);
1175        self.emit(RoderEvent::AgentSwarmModeChanged(
1176            roder_api::subagents::AgentSwarmModeChanged {
1177                thread_id: "runtime".to_string(),
1178                turn_id: None,
1179                enabled,
1180                trigger,
1181                timestamp: OffsetDateTime::now_utc(),
1182            },
1183        ))
1184        .await;
1185        Ok(next)
1186    }
1187
1188    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1189    /// stored per thread so toggling swarm mode on one thread does not leak the
1190    /// reminder into other threads sharing the runtime; absent threads fall back
1191    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1192    /// carries the real `thread_id`.
1193    pub async fn set_agent_swarm_mode_for_thread(
1194        &self,
1195        thread_id: &str,
1196        enabled: bool,
1197        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1198    ) -> bool {
1199        {
1200            let mut modes = self.agent_swarm_modes.write().await;
1201            modes.insert(thread_id.to_string(), enabled);
1202        }
1203        self.emit(RoderEvent::AgentSwarmModeChanged(
1204            roder_api::subagents::AgentSwarmModeChanged {
1205                thread_id: thread_id.to_string(),
1206                turn_id: None,
1207                enabled,
1208                trigger,
1209                timestamp: OffsetDateTime::now_utc(),
1210            },
1211        ))
1212        .await;
1213        enabled
1214    }
1215
1216    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1217    /// override when present, otherwise the runtime-global default. The turn loop
1218    /// uses this to decide whether to inject the swarm reminder.
1219    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1220        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1221            return enabled;
1222        }
1223        self.status().await.agent_swarm_mode
1224    }
1225
1226    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1227    /// policy is injected into each turn's developer instructions for any
1228    /// model, so every app-server/SDK client gets it (not only the TUI).
1229    pub async fn set_ultra_mode(
1230        &self,
1231        enabled: bool,
1232        trigger: roder_api::subagents::UltraModeTrigger,
1233    ) -> anyhow::Result<RuntimeConfig> {
1234        let mut cfg = self.config.write().await;
1235        cfg.ultra_mode = enabled;
1236        let next = cfg.clone();
1237        drop(cfg);
1238        self.emit(RoderEvent::UltraModeChanged(
1239            roder_api::subagents::UltraModeChanged {
1240                thread_id: "runtime".to_string(),
1241                turn_id: None,
1242                enabled,
1243                trigger,
1244                timestamp: OffsetDateTime::now_utc(),
1245            },
1246        ))
1247        .await;
1248        Ok(next)
1249    }
1250
1251    /// Toggle ultra mode for a single thread. The override is stored per thread
1252    /// so toggling on one thread does not leak proactive multi-agent policy into
1253    /// other threads sharing the runtime; absent threads fall back to the
1254    /// runtime-global default.
1255    pub async fn set_ultra_mode_for_thread(
1256        &self,
1257        thread_id: &str,
1258        enabled: bool,
1259        trigger: roder_api::subagents::UltraModeTrigger,
1260    ) -> bool {
1261        {
1262            let mut modes = self.ultra_modes.write().await;
1263            modes.insert(thread_id.to_string(), enabled);
1264        }
1265        self.emit(RoderEvent::UltraModeChanged(
1266            roder_api::subagents::UltraModeChanged {
1267                thread_id: thread_id.to_string(),
1268                turn_id: None,
1269                enabled,
1270                trigger,
1271                timestamp: OffsetDateTime::now_utc(),
1272            },
1273        ))
1274        .await;
1275        enabled
1276    }
1277
1278    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1279    /// override when present, otherwise the runtime-global default. The turn
1280    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1281    /// to inject proactive multi-agent policy.
1282    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1283        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1284            return enabled;
1285        }
1286        self.status().await.ultra_mode
1287    }
1288
1289    pub async fn set_hosted_web_search(
1290        &self,
1291        mode: HostedWebSearchMode,
1292    ) -> anyhow::Result<RuntimeConfig> {
1293        let mut cfg = self.config.write().await;
1294        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1295        Ok(cfg.clone())
1296    }
1297
1298    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1299        let gate = DefaultPolicyGate::new();
1300        let mut pending = self.pending_tool_approvals.lock().await;
1301        let approval_ids = pending
1302            .iter()
1303            .filter_map(|(approval_id, approval)| {
1304                let ctx = ToolExecutionContext::new(
1305                    approval.thread_id.clone(),
1306                    approval.turn_id.clone(),
1307                    mode,
1308                );
1309                matches!(
1310                    gate.decide(&approval.call, mode, &ctx),
1311                    PolicyDecision::AutoApproved { .. }
1312                )
1313                .then_some(approval_id.clone())
1314            })
1315            .collect::<Vec<_>>();
1316        let approvals = approval_ids
1317            .into_iter()
1318            .filter_map(|approval_id| {
1319                pending
1320                    .remove(&approval_id)
1321                    .map(|approval| (approval_id, approval))
1322            })
1323            .collect::<Vec<_>>();
1324        drop(pending);
1325
1326        for (approval_id, approval) in approvals {
1327            let ctx = ToolExecutionContext::new(
1328                approval.thread_id.clone(),
1329                approval.turn_id.clone(),
1330                mode,
1331            );
1332            let decision = gate.decide(&approval.call, mode, &ctx);
1333            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1334                thread_id: approval.thread_id.clone(),
1335                turn_id: approval.turn_id.clone(),
1336                tool_id: approval.tool_id.clone(),
1337                tool_name: approval.tool_name.clone(),
1338                mode,
1339                decision,
1340                timestamp: OffsetDateTime::now_utc(),
1341            }))
1342            .await;
1343            if mode == PolicyMode::Bypass {
1344                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1345                    thread_id: approval.thread_id.clone(),
1346                    turn_id: approval.turn_id.clone(),
1347                    tool_id: approval.tool_id.clone(),
1348                    tool_name: approval.tool_name.clone(),
1349                    timestamp: OffsetDateTime::now_utc(),
1350                }))
1351                .await;
1352            }
1353            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1354                thread_id: approval.thread_id,
1355                turn_id: approval.turn_id,
1356                approval_id,
1357                tool_id: approval.tool_id,
1358                tool_name: approval.tool_name,
1359                approved: true,
1360                timestamp: OffsetDateTime::now_utc(),
1361            }))
1362            .await;
1363            let _ = approval.tx.send(true);
1364        }
1365    }
1366
1367    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1368        *self.pending_plan_exit.write().await = Some(pending.clone());
1369        self.emit(RoderEvent::PolicyExitPlanRequested(
1370            PolicyExitPlanRequested {
1371                thread_id: pending.thread_id,
1372                turn_id: pending.turn_id,
1373                request_id: pending.request_id,
1374                target_mode: pending.target_mode,
1375                plan_summary: pending.plan_summary,
1376                next_steps: pending.next_steps,
1377                timestamp: OffsetDateTime::now_utc(),
1378            },
1379        ))
1380        .await;
1381    }
1382
1383    pub async fn resolve_pending_plan_exit(
1384        &self,
1385        request_id: &str,
1386        approved: bool,
1387    ) -> anyhow::Result<Option<PendingPlanExit>> {
1388        let mut pending = self.pending_plan_exit.write().await;
1389        let Some(current) = pending.clone() else {
1390            return Ok(None);
1391        };
1392        if current.request_id != request_id {
1393            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1394        }
1395        *pending = None;
1396        drop(pending);
1397
1398        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1399        let resolved_mode = if approved {
1400            let mut cfg = self.config.write().await;
1401            let previous_mode = cfg.policy_mode;
1402            cfg.policy_mode = current.target_mode;
1403            drop(cfg);
1404            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1405                thread_id: current.thread_id.clone(),
1406                turn_id: Some(current.turn_id.clone()),
1407                previous_mode,
1408                new_mode: current.target_mode,
1409                reason: Some("approved plan exit".to_string()),
1410                timestamp: OffsetDateTime::now_utc(),
1411            }))
1412            .await;
1413            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1414                .await;
1415            current.target_mode
1416        } else {
1417            self.status().await.policy_mode
1418        };
1419        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1420            .await;
1421        Ok(Some(current))
1422    }
1423
1424    pub async fn resolve_tool_approval(
1425        &self,
1426        approval_id: &str,
1427        approved: bool,
1428    ) -> anyhow::Result<bool> {
1429        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1430        let Some(pending) = pending else {
1431            return Ok(false);
1432        };
1433        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1434            thread_id: pending.thread_id,
1435            turn_id: pending.turn_id,
1436            approval_id: approval_id.to_string(),
1437            tool_id: pending.tool_id,
1438            tool_name: pending.tool_name,
1439            approved,
1440            timestamp: OffsetDateTime::now_utc(),
1441        }))
1442        .await;
1443        let _ = pending.tx.send(approved);
1444        Ok(true)
1445    }
1446
1447    pub async fn request_app_server_tool_approval(
1448        &self,
1449        call: ToolCall,
1450        reason: Option<String>,
1451    ) -> anyhow::Result<bool> {
1452        let approval_id = call.id.clone();
1453        let (tx, rx) = oneshot::channel();
1454        self.pending_tool_approvals.lock().await.insert(
1455            approval_id.clone(),
1456            PendingToolApproval {
1457                thread_id: call.thread_id.clone(),
1458                turn_id: call.turn_id.clone(),
1459                tool_id: call.id.clone(),
1460                tool_name: call.name.clone(),
1461                call: call.clone(),
1462                tx,
1463            },
1464        );
1465        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1466            thread_id: call.thread_id.clone(),
1467            turn_id: call.turn_id.clone(),
1468            approval_id,
1469            tool_id: call.id.clone(),
1470            tool_name: call.name.clone(),
1471            reason,
1472            timestamp: OffsetDateTime::now_utc(),
1473        }))
1474        .await;
1475        Ok(rx.await.unwrap_or(false))
1476    }
1477
1478    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1479    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1480    pub async fn resolve_external_tool_call(
1481        &self,
1482        request_id: &str,
1483        resolution: ExternalToolResolution,
1484    ) -> anyhow::Result<bool> {
1485        let pending = self
1486            .pending_external_tool_calls
1487            .lock()
1488            .await
1489            .remove(request_id);
1490        let Some(pending) = pending else {
1491            return Ok(false);
1492        };
1493        self.emit(RoderEvent::ExternalToolCallResolved(
1494            ExternalToolCallResolved {
1495                thread_id: pending.thread_id,
1496                turn_id: pending.turn_id,
1497                request_id: request_id.to_string(),
1498                tool_id: pending.tool_id,
1499                tool_name: pending.tool_name,
1500                outcome: ExternalToolCallOutcome::Resolved,
1501                is_error: resolution.is_error,
1502                timestamp: OffsetDateTime::now_utc(),
1503            },
1504        ))
1505        .await;
1506        let _ = pending.tx.send(resolution);
1507        Ok(true)
1508    }
1509
1510    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1511    /// interrupt does not leak requests waiting on `tools/resolve`.
1512    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1513        let cancelled = {
1514            let mut pending = self.pending_external_tool_calls.lock().await;
1515            let request_ids = pending
1516                .iter()
1517                .filter(|(_, call)| &call.turn_id == turn_id)
1518                .map(|(request_id, _)| request_id.clone())
1519                .collect::<Vec<_>>();
1520            request_ids
1521                .into_iter()
1522                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1523                .collect::<Vec<_>>()
1524        };
1525        for (request_id, call) in cancelled {
1526            self.emit(RoderEvent::ExternalToolCallResolved(
1527                ExternalToolCallResolved {
1528                    thread_id: call.thread_id,
1529                    turn_id: call.turn_id,
1530                    request_id,
1531                    tool_id: call.tool_id,
1532                    tool_name: call.tool_name,
1533                    outcome: ExternalToolCallOutcome::Cancelled,
1534                    is_error: true,
1535                    timestamp: OffsetDateTime::now_utc(),
1536                },
1537            ))
1538            .await;
1539        }
1540    }
1541
1542    pub async fn resolve_user_input(
1543        &self,
1544        request_id: &str,
1545        answers: serde_json::Value,
1546    ) -> anyhow::Result<bool> {
1547        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1548        let Some(pending) = pending else {
1549            return Ok(false);
1550        };
1551        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1552            thread_id: pending.thread_id,
1553            turn_id: pending.turn_id,
1554            request_id: request_id.to_string(),
1555            answers: answers.clone(),
1556            timestamp: OffsetDateTime::now_utc(),
1557        }))
1558        .await;
1559        let _ = pending.tx.send(answers);
1560        Ok(true)
1561    }
1562
1563    async fn emit_plan_exit_resolved(
1564        &self,
1565        current: &PendingPlanExit,
1566        approved: bool,
1567        resolved_mode: PolicyMode,
1568    ) {
1569        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1570            thread_id: current.thread_id.clone(),
1571            turn_id: current.turn_id.clone(),
1572            request_id: current.request_id.clone(),
1573            approved,
1574            target_mode: current.target_mode,
1575            resolved_mode,
1576            timestamp: OffsetDateTime::now_utc(),
1577        }))
1578        .await;
1579    }
1580
1581    pub async fn select_provider(
1582        &self,
1583        provider: String,
1584        model: Option<String>,
1585        reasoning: Option<String>,
1586    ) -> anyhow::Result<RuntimeConfig> {
1587        let next = self
1588            .preview_provider_selection(provider, model, reasoning)
1589            .await?;
1590        let mut cfg = self.config.write().await;
1591        *cfg = next;
1592        Ok(cfg.clone())
1593    }
1594
1595    pub async fn preview_provider_selection(
1596        &self,
1597        provider: String,
1598        model: Option<String>,
1599        reasoning: Option<String>,
1600    ) -> anyhow::Result<RuntimeConfig> {
1601        self.engine_for(&provider)?;
1602        let mut cfg = self.config.read().await.clone();
1603        cfg.default_provider = provider;
1604        if let Some(model) = model {
1605            cfg.default_model = model;
1606        }
1607        if let Some(reasoning) = reasoning {
1608            if reasoning == REASONING_NONE
1609                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1610            {
1611                return Ok(cfg.clone());
1612            }
1613            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1614            cfg.reasoning = Some(reasoning);
1615        }
1616        Ok(cfg)
1617    }
1618
1619    pub async fn effective_reasoning(&self) -> String {
1620        let cfg = self.config.read().await;
1621        effective_reasoning_for_model(&cfg, &cfg.default_model)
1622    }
1623
1624    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1625        effective_reasoning_for_model(cfg, &cfg.default_model)
1626    }
1627
1628    pub async fn set_dynamic_workflow_effort(
1629        &self,
1630        effort_profile: DynamicWorkflowEffortProfile,
1631    ) -> RuntimeConfig {
1632        let mut cfg = self.config.write().await;
1633        cfg.dynamic_workflows.effort_profile = effort_profile;
1634        cfg.clone()
1635    }
1636
1637    pub async fn dynamic_workflow_trigger_decision(
1638        &self,
1639        message: &str,
1640    ) -> WorkflowTriggerDecision {
1641        let cfg = self.config.read().await;
1642        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1643    }
1644
1645    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1646        self.create_thread_with(CreateThreadRequest {
1647            title,
1648            workspace: self.workspace.display().to_string(),
1649            workspace_id: None,
1650            root_id: None,
1651            provider: None,
1652            model: None,
1653            selection_mode: None,
1654            tool_allowlist: Vec::new(),
1655            developer_instructions: None,
1656            external_tools: Vec::new(),
1657            runner: None,
1658        })
1659        .await
1660    }
1661
1662    /**
1663     * Resolves an explicit thread-runner selection into a persisted binding.
1664     * Fails fast at thread creation when the provider is missing or rejects
1665     * the destination, instead of surfacing the error on the first tool call.
1666     */
1667    async fn resolve_thread_runner_binding(
1668        &self,
1669        thread_id: &str,
1670        selection: ThreadRunnerSelection,
1671    ) -> anyhow::Result<ThreadRunnerBinding> {
1672        let provider = self
1673            .registry
1674            .remote_runner_providers
1675            .iter()
1676            .find(|provider| provider.id() == selection.provider_id)
1677            .cloned()
1678            .ok_or_else(|| {
1679                anyhow::anyhow!(
1680                    "remote runner provider {:?} is not installed",
1681                    selection.provider_id
1682                )
1683            })?;
1684        let workspace = selection.workspace.trim();
1685        anyhow::ensure!(
1686            std::path::Path::new(workspace).is_absolute(),
1687            "runner workspace must be an absolute path on the runner: {workspace:?}"
1688        );
1689        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1690        for read_root in &selection.read_roots {
1691            let trimmed = read_root.trim();
1692            anyhow::ensure!(
1693                std::path::Path::new(trimmed).is_absolute(),
1694                "runner read root must be an absolute path on the runner: {trimmed:?}"
1695            );
1696            read_roots.push(PathBuf::from(trimmed));
1697        }
1698        let destination = RunnerDestination {
1699            id: format!("thread-{thread_id}"),
1700            provider_id: selection.provider_id,
1701            config: selection.config,
1702            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1703        };
1704        provider.validate_destination(&destination).await?;
1705        Ok(ThreadRunnerBinding {
1706            destination,
1707            workspace: PathBuf::from(workspace),
1708            read_roots,
1709        })
1710    }
1711
1712    /**
1713     * Build a per-thread binding from a runtime-level destination (selected via
1714     * the TUI runner picker or config `default_destination`) when the
1715     * destination's provider advertises a default workspace. Returns `None` when
1716     * there is no destination or the provider opts out (no default workspace),
1717     * preserving the legacy behavior where such threads keep local tools.
1718     */
1719    async fn synthesize_runtime_runner_binding(
1720        &self,
1721        thread_id: &str,
1722        destination: Option<RunnerDestination>,
1723    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1724        let Some(destination) = destination else {
1725            return Ok(None);
1726        };
1727        let Some(provider) = self
1728            .registry
1729            .remote_runner_providers
1730            .iter()
1731            .find(|provider| provider.id() == destination.provider_id)
1732        else {
1733            return Ok(None);
1734        };
1735        // An explicit workspace in the destination config wins over the
1736        // provider default; absence of both means the provider opts out.
1737        let workspace = destination
1738            .config
1739            .get("working_dir")
1740            .and_then(serde_json::Value::as_str)
1741            .map(str::to_string)
1742            .or_else(|| provider.default_workspace());
1743        let Some(workspace) = workspace else {
1744            return Ok(None);
1745        };
1746        let selection = ThreadRunnerSelection {
1747            provider_id: destination.provider_id.clone(),
1748            config: destination.config.clone(),
1749            workspace,
1750            read_roots: Vec::new(),
1751        };
1752        Ok(Some(
1753            self.resolve_thread_runner_binding(thread_id, selection)
1754                .await?,
1755        ))
1756    }
1757
1758    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1759    pub async fn validate_thread_runner_selection(
1760        &self,
1761        selection: ThreadRunnerSelection,
1762    ) -> anyhow::Result<()> {
1763        self.resolve_thread_runner_binding("validate", selection)
1764            .await
1765            .map(|_| ())
1766    }
1767
1768    pub async fn create_thread_with(
1769        &self,
1770        req: CreateThreadRequest,
1771    ) -> anyhow::Result<ThreadMetadata> {
1772        let cfg = self.config.read().await.clone();
1773        let now = OffsetDateTime::now_utc();
1774        let workspace = validate_thread_workspace(&req.workspace)?;
1775        let provider = req.provider.unwrap_or(cfg.default_provider);
1776        let model = req.model.unwrap_or(cfg.default_model);
1777        let selection_mode = req
1778            .selection_mode
1779            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1780        let thread_id = uuid::Uuid::new_v4().to_string();
1781        let runner_binding = match req.runner {
1782            Some(selection) => Some(
1783                self.resolve_thread_runner_binding(&thread_id, selection)
1784                    .await?,
1785            ),
1786            // No explicit per-thread selection: if a runtime-level destination
1787            // is active and its provider advertises a default workspace, bind
1788            // the new thread so its coding tools route into the runner. This is
1789            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1790            // execute tools in the sandbox instead of locally. Providers that
1791            // return no default workspace stay unbound (legacy local-tools).
1792            None => {
1793                self.synthesize_runtime_runner_binding(
1794                    &thread_id,
1795                    cfg.remote_runner_destination.clone(),
1796                )
1797                .await?
1798            }
1799        };
1800        let runner_destination = runner_binding
1801            .as_ref()
1802            .map(|binding| binding.destination.clone())
1803            .or_else(|| cfg.remote_runner_destination.clone());
1804        let metadata = ThreadMetadata {
1805            thread_id,
1806            title: req.title,
1807            workspace,
1808            workspace_id: req.workspace_id,
1809            root_id: req.root_id,
1810            provider: Some(provider),
1811            model: Some(model),
1812            selection_mode: Some(selection_mode),
1813            tool_allowlist: req.tool_allowlist,
1814            developer_instructions: req.developer_instructions,
1815            external_tools: req.external_tools,
1816            runner_destination,
1817            runner_state: None,
1818            runner_binding,
1819            created_at: now,
1820            updated_at: now,
1821            message_count: 0,
1822            usage: None,
1823            parent_thread_id: None,
1824            forked_from_turn_id: None,
1825            workspace_fork: None,
1826        };
1827
1828        let metadata = if let Some(store) = &self.thread_store {
1829            store.create_thread(metadata).await?
1830        } else {
1831            metadata
1832        };
1833        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1834            thread_id: metadata.thread_id.clone(),
1835            timestamp: OffsetDateTime::now_utc(),
1836        }))
1837        .await;
1838        Ok(metadata)
1839    }
1840
1841    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1842        if let Some(store) = &self.thread_store {
1843            return store.list_threads().await;
1844        }
1845        Ok(Vec::new())
1846    }
1847
1848    pub async fn list_threads_page(
1849        &self,
1850        options: roder_api::thread::ThreadListOptions,
1851    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1852        if let Some(store) = &self.thread_store {
1853            return store.list_threads_page(options).await;
1854        }
1855        Ok(roder_api::thread::ThreadListPage::default())
1856    }
1857
1858    pub async fn load_thread_metadata(
1859        &self,
1860        thread_id: &str,
1861    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1862        if let Some(store) = &self.thread_store {
1863            return store.load_thread_metadata(&thread_id.to_string()).await;
1864        }
1865        Ok(None)
1866    }
1867
1868    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1869        let archived = if let Some(store) = &self.thread_store {
1870            store.archive_thread(&thread_id.to_string()).await?
1871        } else {
1872            false
1873        };
1874        if archived {
1875            let workspace = self.config.read().await.workspace.clone();
1876            crate::hooks::run_lifecycle(
1877                self,
1878                &thread_id.to_string(),
1879                &SESSION_HOOK_TURN_ID.to_string(),
1880                workspace.as_deref(),
1881                "SessionEnd",
1882                Some("clear"),
1883                serde_json::json!({"reason": "archive"}),
1884            )
1885            .await;
1886            self.thread_item_cache
1887                .lock()
1888                .await
1889                .remove_thread(&thread_id.to_string());
1890            self.evict_runner_session(&thread_id.to_string()).await;
1891        }
1892        Ok(archived)
1893    }
1894
1895    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1896        let cfg = self.config.read().await.clone();
1897        let workspace = self.workspace.display().to_string();
1898        let lead_thread_id = match req.lead_thread_id {
1899            Some(thread_id) => thread_id,
1900            None => {
1901                self.create_thread_with(CreateThreadRequest {
1902                    title: Some("Team lead".to_string()),
1903                    workspace: workspace.clone(),
1904                    workspace_id: None,
1905                    root_id: None,
1906                    provider: None,
1907                    model: None,
1908                    selection_mode: None,
1909                    tool_allowlist: Vec::new(),
1910                    developer_instructions: None,
1911                    external_tools: Vec::new(),
1912                    runner: None,
1913                })
1914                .await?
1915                .thread_id
1916            }
1917        };
1918        let team_id = uuid::Uuid::new_v4().to_string();
1919        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1920        let lead_selection = match active_lead_turn_id.as_ref() {
1921            Some(turn_id) => self
1922                .active_turn_selections
1923                .read()
1924                .await
1925                .get(turn_id)
1926                .cloned(),
1927            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1928        };
1929        let lead_concrete_selection = lead_selection
1930            .as_ref()
1931            .map(ModelSelectionMode::concrete_selection);
1932        let mut lead = crate::teams::lead_member(
1933            lead_thread_id.clone(),
1934            lead_concrete_selection
1935                .as_ref()
1936                .map(|selection| selection.provider.clone())
1937                .or_else(|| Some(cfg.default_provider.clone())),
1938            lead_concrete_selection
1939                .as_ref()
1940                .map(|selection| selection.model.clone())
1941                .or_else(|| Some(cfg.default_model.clone())),
1942            cfg.policy_mode,
1943        );
1944        if let Some(turn_id) = active_lead_turn_id {
1945            lead.current_turn_id = Some(turn_id);
1946            lead.status = TeamMemberStatus::Running;
1947        }
1948        let mut members = vec![lead];
1949
1950        for (index, member) in req.members.into_iter().enumerate() {
1951            let thread = self
1952                .create_thread_with(CreateThreadRequest {
1953                    title: Some(member.name.clone()),
1954                    workspace: workspace.clone(),
1955                    workspace_id: None,
1956                    root_id: None,
1957                    provider: member.model_provider.clone(),
1958                    model: member.model.clone(),
1959                    selection_mode: None,
1960                    tool_allowlist: Vec::new(),
1961                    developer_instructions: None,
1962                    external_tools: Vec::new(),
1963                    runner: None,
1964                })
1965                .await?;
1966            let member_id = format!("member-{}", index + 1);
1967            let descriptor = crate::teams::teammate_member(
1968                member_id.clone(),
1969                member.name,
1970                thread.thread_id.clone(),
1971                member.model_provider.or(thread.provider),
1972                member.model.or(thread.model),
1973                cfg.policy_mode,
1974            );
1975            members.push(descriptor);
1976        }
1977
1978        let now = OffsetDateTime::now_utc();
1979        let team = self
1980            .teams
1981            .insert(TeamState {
1982                id: team_id.clone(),
1983                lead_thread_id: lead_thread_id.clone(),
1984                display_mode: req.display_mode,
1985                members,
1986                mailbox: Vec::new(),
1987                tasks: Vec::new(),
1988                created_at: now,
1989                updated_at: now,
1990            })
1991            .await?;
1992        self.emit(RoderEvent::TeamStarted(TeamStarted {
1993            team_id: team_id.clone(),
1994            lead_thread_id,
1995            display_mode: team.display_mode,
1996            members: team.members.clone(),
1997            tasks: team.tasks.clone(),
1998            timestamp: OffsetDateTime::now_utc(),
1999        }))
2000        .await;
2001        for member in team
2002            .members
2003            .iter()
2004            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2005        {
2006            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2007                team_id: team_id.clone(),
2008                member: member.clone(),
2009                timestamp: OffsetDateTime::now_utc(),
2010            }))
2011            .await;
2012        }
2013        Ok(team)
2014    }
2015
2016    pub async fn list_teams(&self) -> Vec<TeamState> {
2017        let active_thread_ids = self
2018            .active_turns
2019            .read()
2020            .await
2021            .values()
2022            .map(|handle| handle.thread_id.clone())
2023            .collect::<std::collections::HashSet<_>>();
2024        self.teams
2025            .list()
2026            .await
2027            .into_iter()
2028            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2029            .collect()
2030    }
2031
2032    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2033        let active_thread_ids = self
2034            .active_turns
2035            .read()
2036            .await
2037            .values()
2038            .map(|handle| handle.thread_id.clone())
2039            .collect::<std::collections::HashSet<_>>();
2040        self.teams
2041            .get(team_id)
2042            .await
2043            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2044    }
2045
2046    pub async fn start_team_member(
2047        &self,
2048        team_id: &str,
2049        req: TeamMemberStartRequest,
2050    ) -> anyhow::Result<TeamState> {
2051        self.start_team_member_with_selection(team_id, req, None)
2052            .await
2053    }
2054
2055    pub async fn message_team_member(
2056        self: &Arc<Self>,
2057        team_id: &str,
2058        member_id: &str,
2059        message: String,
2060    ) -> anyhow::Result<TurnId> {
2061        let team = self
2062            .read_team(team_id)
2063            .await
2064            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2065        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2066            .await
2067    }
2068
2069    /// Queue a mailbox message without starting an idle agent. If the target is
2070    /// already running, the message is delivered at the next inference boundary.
2071    pub(crate) async fn queue_team_member_message(
2072        self: &Arc<Self>,
2073        caller_thread_id: &ThreadId,
2074        team_id: &str,
2075        member_id: &str,
2076        message: String,
2077    ) -> anyhow::Result<Option<TurnId>> {
2078        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2079        let team = self
2080            .read_team(team_id)
2081            .await
2082            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2083        let member = team
2084            .members
2085            .iter()
2086            .find(|member| member.id == member_id)
2087            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2088            .clone();
2089        if member.status == TeamMemberStatus::Closed {
2090            anyhow::bail!("subagent {} is closed", member.name);
2091        }
2092        let from_member_id = team
2093            .members
2094            .iter()
2095            .find(|candidate| candidate.thread_id == *caller_thread_id)
2096            .map(|candidate| candidate.id.clone());
2097        self.teams
2098            .append_mailbox_message(
2099                team_id,
2100                from_member_id,
2101                member_id.to_string(),
2102                TeamMailboxMessageKind::Message,
2103                message,
2104            )
2105            .await?;
2106        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2107            return Ok(None);
2108        };
2109        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2110            .await?;
2111        Ok(Some(turn_id))
2112    }
2113
2114    /// Deliver queued messages and start an idle agent, or steer its active turn.
2115    pub(crate) async fn followup_team_member(
2116        self: &Arc<Self>,
2117        caller_thread_id: &ThreadId,
2118        team_id: &str,
2119        member_id: &str,
2120        message: String,
2121    ) -> anyhow::Result<TurnId> {
2122        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2123        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2124            .await
2125    }
2126
2127    async fn followup_team_member_locked(
2128        self: &Arc<Self>,
2129        caller_thread_id: &ThreadId,
2130        team_id: &str,
2131        member_id: &str,
2132        message: String,
2133    ) -> anyhow::Result<TurnId> {
2134        let team = self
2135            .read_team(team_id)
2136            .await
2137            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2138        let member = team
2139            .members
2140            .iter()
2141            .find(|member| member.id == member_id)
2142            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2143            .clone();
2144        if member.status == TeamMemberStatus::Closed {
2145            anyhow::bail!("subagent {} is closed", member.name);
2146        }
2147        let from_member_id = team
2148            .members
2149            .iter()
2150            .find(|candidate| candidate.thread_id == *caller_thread_id)
2151            .map(|candidate| candidate.id.clone());
2152        self.teams
2153            .append_mailbox_message(
2154                team_id,
2155                from_member_id,
2156                member_id.to_string(),
2157                TeamMailboxMessageKind::NewTask,
2158                message,
2159            )
2160            .await?;
2161        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2162            self.deliver_pending_team_mailbox(
2163                team_id,
2164                member_id,
2165                member.thread_id,
2166                turn_id.clone(),
2167            )
2168            .await?;
2169            return Ok(turn_id);
2170        }
2171
2172        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2173            .await?;
2174        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2175        let inherited_context = self
2176            .team_member_turn_contexts
2177            .lock()
2178            .await
2179            .get(&member.thread_id)
2180            .cloned();
2181        let workspace = inherited_context
2182            .as_ref()
2183            .map(|context| context.workspace.clone())
2184            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2185            .unwrap_or_else(|| self.workspace.display().to_string());
2186        let member_thread_id = member.thread_id;
2187        let turn_id = self
2188            .start_turn(StartTurnRequest {
2189                thread_id: member_thread_id.clone(),
2190                message: String::new(),
2191                images: Vec::new(),
2192                provider_override: member.model_provider,
2193                model_override: member.model,
2194                reasoning_override: metadata
2195                    .as_ref()
2196                    .and_then(|metadata| metadata.selection_mode.as_ref())
2197                    .and_then(ModelSelectionMode::reasoning)
2198                    .map(str::to_string),
2199                workspace,
2200                instructions: inherited_context
2201                    .as_ref()
2202                    .map(|context| context.instructions.clone())
2203                    .unwrap_or_else(crate::default_instructions),
2204                developer_context: inherited_context
2205                    .as_ref()
2206                    .and_then(|context| context.developer_context.clone()),
2207                task_ledger_required: false,
2208            })
2209            .await?;
2210        Ok(turn_id)
2211    }
2212
2213    pub async fn set_team_member_policy_mode(
2214        &self,
2215        team_id: &str,
2216        member_id: &str,
2217        policy_mode: PolicyMode,
2218    ) -> anyhow::Result<TeamState> {
2219        self.teams
2220            .set_member_policy_mode(team_id, member_id, policy_mode)
2221            .await
2222    }
2223
2224    pub async fn interrupt_team_member(
2225        &self,
2226        team_id: &str,
2227        member_id: &str,
2228    ) -> anyhow::Result<Option<TurnId>> {
2229        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2230        let team = self
2231            .read_team(team_id)
2232            .await
2233            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2234        let member = team
2235            .members
2236            .iter()
2237            .find(|member| member.id == member_id)
2238            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2239            .clone();
2240        if member.status != TeamMemberStatus::Running {
2241            return Ok(None);
2242        }
2243        let Some(turn_id) = member.current_turn_id.clone() else {
2244            return Ok(None);
2245        };
2246        if self
2247            .active_turn_for_thread(&member.thread_id)
2248            .await
2249            .as_ref()
2250            != Some(&turn_id)
2251        {
2252            return Ok(None);
2253        }
2254        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2255            .await?;
2256        // Make queued mailbox work immediately eligible for the replacement turn while the
2257        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2258        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2259        // reservation acquired by its replacement.
2260        self.teams
2261            .release_mailbox_reservations_for_turn(&turn_id)
2262            .await;
2263        self.teams
2264            .update_member(team_id, member_id, |member| {
2265                member.status = TeamMemberStatus::Interrupted;
2266                member.current_turn_id = None;
2267                member.terminal_error = None;
2268            })
2269            .await?;
2270        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2271            team_id: team_id.to_string(),
2272            member_id: member_id.to_string(),
2273            member_thread_id: member.thread_id,
2274            turn_id: Some(turn_id.clone()),
2275            status: TeamMemberStatus::Interrupted,
2276            final_message: member.final_message,
2277            error: None,
2278            timestamp: OffsetDateTime::now_utc(),
2279        }))
2280        .await;
2281        Ok(Some(turn_id))
2282    }
2283
2284    pub async fn close_team_member(
2285        &self,
2286        team_id: &str,
2287        member_id: &str,
2288    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2289        let team = self
2290            .read_team(team_id)
2291            .await
2292            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2293        let member = team
2294            .members
2295            .iter()
2296            .find(|member| member.id == member_id)
2297            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2298            .clone();
2299        if member.role == roder_api::teams::TeamMemberRole::Lead {
2300            anyhow::bail!("team lead cannot be closed as a subagent");
2301        }
2302        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2303            if let Some(turn_id) = member.current_turn_id.clone() {
2304                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2305                    .await?;
2306                Some(turn_id)
2307            } else {
2308                None
2309            }
2310        } else {
2311            member.current_turn_id.clone()
2312        };
2313        let updated = self
2314            .teams
2315            .update_member(team_id, member_id, |member| {
2316                member.status = TeamMemberStatus::Closed;
2317                member.current_turn_id = None;
2318            })
2319            .await?;
2320        let closed = updated
2321            .members
2322            .iter()
2323            .find(|member| member.id == member_id)
2324            .cloned()
2325            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2326        self.team_member_turn_contexts
2327            .lock()
2328            .await
2329            .remove(&closed.thread_id);
2330        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2331            team_id: team_id.to_string(),
2332            member_id: closed.id.clone(),
2333            member_thread_id: closed.thread_id.clone(),
2334            turn_id: interrupted_turn_id,
2335            status: TeamMemberStatus::Closed,
2336            final_message: closed.final_message.clone(),
2337            error: closed.terminal_error.clone(),
2338            timestamp: OffsetDateTime::now_utc(),
2339        }))
2340        .await;
2341        Ok(closed)
2342    }
2343
2344    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2345        let Some(team) = self.read_team(team_id).await else {
2346            return Ok(false);
2347        };
2348        if !force
2349            && team
2350                .members
2351                .iter()
2352                .any(|member| member.status == TeamMemberStatus::Running)
2353        {
2354            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2355        }
2356        if force {
2357            for member in team.members.iter().filter(|member| {
2358                member.role != roder_api::teams::TeamMemberRole::Lead
2359                    && member.status == TeamMemberStatus::Running
2360            }) {
2361                if let Some(turn_id) = member
2362                    .current_turn_id
2363                    .clone()
2364                    .or(self.active_turn_for_thread(&member.thread_id).await)
2365                {
2366                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2367                }
2368            }
2369        }
2370        let removed = self.teams.remove(team_id).await?.is_some();
2371        if removed {
2372            let member_thread_ids = team
2373                .members
2374                .iter()
2375                .map(|member| member.thread_id.clone())
2376                .collect::<std::collections::HashSet<_>>();
2377            self.team_member_turn_contexts
2378                .lock()
2379                .await
2380                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2381            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2382                team_id: team_id.to_string(),
2383                forced: force,
2384                timestamp: OffsetDateTime::now_utc(),
2385            }))
2386            .await;
2387        }
2388        Ok(removed)
2389    }
2390
2391    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2392        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2393            return mode;
2394        }
2395        self.status().await.policy_mode
2396    }
2397
2398    async fn complete_team_member_turn_with_result(
2399        &self,
2400        thread_id: &ThreadId,
2401        turn_id: &TurnId,
2402        status: TeamMemberStatus,
2403        final_message: Option<String>,
2404        terminal_error: Option<String>,
2405    ) -> anyhow::Result<()> {
2406        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2407        let Some((team_id, member)) = self
2408            .teams
2409            .complete_member_turn(
2410                thread_id,
2411                turn_id,
2412                status,
2413                final_message.clone(),
2414                terminal_error.clone(),
2415            )
2416            .await?
2417        else {
2418            return Ok(());
2419        };
2420        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2421            && let Some(team) = self.read_team(&team_id).await
2422            && let Some(parent) = team
2423                .members
2424                .iter()
2425                .find(|candidate| candidate.thread_id == *parent_thread_id)
2426        {
2427            let identity = member
2428                .agent_path
2429                .as_deref()
2430                .or(member.task_name.as_deref())
2431                .unwrap_or(&member.name);
2432            let mut report = format!("Agent {identity} finished with status {status:?}.");
2433            if let Some(message) = final_message.as_deref()
2434                && !message.trim().is_empty()
2435            {
2436                report.push_str("\n\nFinal result:\n");
2437                report.push_str(message);
2438            }
2439            if let Some(error) = terminal_error.as_deref()
2440                && !error.trim().is_empty()
2441            {
2442                report.push_str("\n\nTerminal error:\n");
2443                report.push_str(error);
2444            }
2445            let mailbox_appended = self
2446                .teams
2447                .append_mailbox_message(
2448                    &team_id,
2449                    Some(member.id.clone()),
2450                    parent.id.clone(),
2451                    TeamMailboxMessageKind::FinalAnswer,
2452                    report,
2453                )
2454                .await
2455                .is_ok();
2456            if mailbox_appended
2457                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2458            {
2459                // The parent may finish between the active-turn lookup and enqueue. Its durable
2460                // mailbox entry remains pending for the next turn, while terminal observers must
2461                // still receive TeamMemberCompleted for this child.
2462                let _ = self
2463                    .deliver_pending_team_mailbox(
2464                        &team_id,
2465                        &parent.id,
2466                        parent_thread_id.clone(),
2467                        parent_turn_id,
2468                    )
2469                    .await;
2470            }
2471        }
2472        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2473            team_id,
2474            member_id: member.id,
2475            member_thread_id: member.thread_id,
2476            turn_id: Some(turn_id.clone()),
2477            status,
2478            final_message,
2479            error: terminal_error,
2480            timestamp: OffsetDateTime::now_utc(),
2481        }))
2482        .await;
2483        Ok(())
2484    }
2485
2486    /// Stops accepting new turns, requests interruption for every active turn,
2487    /// and waits for their runtime tasks to reach terminal cleanup up to
2488    /// `timeout`. Repeated calls are safe and continue draining the same set.
2489    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2490        let started_at = tokio::time::Instant::now();
2491        let persistence_failures_before =
2492            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2493        let turn_admission = self.turn_admission.lock().await;
2494        self.accepting_turns.store(false, Ordering::Release);
2495        let active = self
2496            .active_turns
2497            .read()
2498            .await
2499            .iter()
2500            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2501            .collect::<Vec<_>>();
2502        let draining = self
2503            .turn_drains
2504            .read()
2505            .await
2506            .iter()
2507            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2508            .collect::<Vec<_>>();
2509        drop(turn_admission);
2510
2511        let mut interrupted_turns = std::collections::BTreeMap::new();
2512        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2513            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2514        }
2515        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2516        for (turn_id, thread_id) in active {
2517            let _ = self
2518                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2519                .await;
2520        }
2521
2522        let wait_for_empty = async {
2523            loop {
2524                let notified = self.active_turns_changed.notified();
2525                if self.active_turns.read().await.is_empty()
2526                    && self.turn_drains.read().await.is_empty()
2527                {
2528                    return;
2529                }
2530                notified.await;
2531            }
2532        };
2533        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2534            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2535                > persistence_failures_before
2536            {
2537                RuntimeDrainOutcome::PersistenceFailed {
2538                    interrupted_turn_ids,
2539                    remaining_turn_ids: Vec::new(),
2540                }
2541            } else {
2542                RuntimeDrainOutcome::Clean {
2543                    interrupted_turn_ids,
2544                }
2545            }
2546        } else {
2547            let active_remaining = self
2548                .active_turns
2549                .read()
2550                .await
2551                .iter()
2552                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2553                .collect::<Vec<_>>();
2554            let draining_remaining = self
2555                .turn_drains
2556                .read()
2557                .await
2558                .iter()
2559                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2560                .collect::<Vec<_>>();
2561            let remaining = active_remaining
2562                .into_iter()
2563                .chain(draining_remaining)
2564                .collect::<std::collections::BTreeMap<_, _>>();
2565            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2566            for turn_id in &remaining_turn_ids {
2567                let handle = remaining
2568                    .get(turn_id)
2569                    .expect("remaining turn ID must have a drain handle");
2570                self.record_turn_lifecycle(
2571                    handle.thread_id.clone(),
2572                    turn_id.clone(),
2573                    TurnLifecycleState::InterruptRequested,
2574                    TurnCleanupState::TimedOut,
2575                    Some(TurnLifecycleReason::Shutdown),
2576                )
2577                .await;
2578            }
2579            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2580                > persistence_failures_before
2581            {
2582                RuntimeDrainOutcome::PersistenceFailed {
2583                    interrupted_turn_ids,
2584                    remaining_turn_ids,
2585                }
2586            } else {
2587                RuntimeDrainOutcome::DeadlineExceeded {
2588                    interrupted_turn_ids,
2589                    remaining_turn_ids,
2590                }
2591            }
2592        };
2593        self.record_lifecycle_drain_outcome(started_at, &outcome);
2594        outcome
2595    }
2596
2597    /// Enables turn submission after a prior drain attempt. This is intended
2598    /// for embedders that keep a runtime alive after a bounded drain timeout.
2599    pub fn resume_accepting_turns(&self) {
2600        self.accepting_turns.store(true, Ordering::Release);
2601    }
2602
2603    pub async fn turn_lifecycle_snapshot(
2604        &self,
2605        thread_id: &ThreadId,
2606    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2607        let Some(store) = &self.thread_store else {
2608            return Ok(TurnLifecycleSnapshot::default());
2609        };
2610        let extension_states = store.load_extension_states(thread_id).await?;
2611        Ok(turn_lifecycle_snapshot(&extension_states))
2612    }
2613
2614    pub async fn load_thread(
2615        &self,
2616        thread_id: &ThreadId,
2617    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2618        let loaded = if let Some(store) = &self.thread_store {
2619            store.load_thread(thread_id).await?
2620        } else {
2621            None
2622        };
2623        if let Some(snapshot) = loaded.as_ref() {
2624            let live_turn_ids = self
2625                .active_turns
2626                .read()
2627                .await
2628                .keys()
2629                .cloned()
2630                .chain(self.turn_drains.read().await.keys().cloned())
2631                .collect::<std::collections::HashSet<_>>();
2632            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2633            for record in lifecycle.records.into_iter().filter(|record| {
2634                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2635            }) {
2636                self.lifecycle_restart_reconciliations
2637                    .fetch_add(1, Ordering::AcqRel);
2638                self.record_turn_lifecycle(
2639                    record.thread_id,
2640                    record.turn_id,
2641                    TurnLifecycleState::RecoveryNeeded,
2642                    TurnCleanupState::Unknown,
2643                    Some(TurnLifecycleReason::RuntimeRestart),
2644                )
2645                .await;
2646            }
2647            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2648                thread_id: thread_id.clone(),
2649                timestamp: OffsetDateTime::now_utc(),
2650            }))
2651            .await;
2652        }
2653        Ok(loaded)
2654    }
2655
2656    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2657        if let Some(store) = &self.thread_store {
2658            let snapshot = store
2659                .load_thread(thread_id)
2660                .await?
2661                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2662            match snapshot {
2663                Ok(snapshot) => {
2664                    if let Some(metadata) = snapshot.metadata {
2665                        // Fork-backed threads fail closed before any write
2666                        // when their workspace was removed out-of-band.
2667                        if let Some(fork) = &metadata.workspace_fork
2668                            && fork.status == roder_api::forks::ForkStatus::Active
2669                            && !std::path::Path::new(&metadata.workspace).is_dir()
2670                        {
2671                            anyhow::bail!(
2672                                "workspace fork {} is missing its workspace at {}; restore it or \
2673                                 remove the fork before running turns in this thread",
2674                                fork.id,
2675                                metadata.workspace
2676                            );
2677                        }
2678                        return Ok(metadata.workspace);
2679                    }
2680                    eprintln!(
2681                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2682                    );
2683                }
2684                Err(err) => {
2685                    eprintln!(
2686                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2687                    );
2688                }
2689            }
2690        }
2691        Ok(self.workspace.display().to_string())
2692    }
2693
2694    pub(crate) async fn selection_mode_for_thread(
2695        &self,
2696        thread_id: &ThreadId,
2697    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2698        let Some(store) = &self.thread_store else {
2699            return Ok(None);
2700        };
2701        Ok(store
2702            .load_thread(thread_id)
2703            .await?
2704            .and_then(|snapshot| snapshot.metadata)
2705            .and_then(|metadata| {
2706                metadata
2707                    .selection_mode
2708                    .or_else(|| match (metadata.provider, metadata.model) {
2709                        (Some(provider), Some(model)) => {
2710                            Some(ModelSelectionMode::manual(provider, model, None))
2711                        }
2712                        _ => None,
2713                    })
2714            }))
2715    }
2716
2717    /// Concrete provider/model for configured-role subagents (`task`,
2718    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2719    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2720    /// startup defaults such as openai/gpt-5.5.
2721    pub(crate) async fn parent_model_selection_for_subagents(
2722        &self,
2723        thread_id: &ThreadId,
2724        turn_id: &TurnId,
2725    ) -> Option<roder_api::inference::ModelSelection> {
2726        if let Some(selection) = self
2727            .active_turn_selections
2728            .read()
2729            .await
2730            .get(turn_id)
2731            .cloned()
2732        {
2733            return Some(selection.concrete_selection());
2734        }
2735        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2736            return Some(selection.concrete_selection());
2737        }
2738        let cfg = self.status().await;
2739        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2740            return None;
2741        }
2742        Some(roder_api::inference::ModelSelection {
2743            provider: cfg.default_provider,
2744            model: cfg.default_model,
2745        })
2746    }
2747
2748    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2749    pub(crate) async fn thread_turn_overrides(
2750        &self,
2751        thread_id: &ThreadId,
2752    ) -> anyhow::Result<ThreadTurnOverrides> {
2753        let Some(store) = &self.thread_store else {
2754            return Ok(ThreadTurnOverrides::default());
2755        };
2756        Ok(store
2757            .load_thread_metadata(thread_id)
2758            .await?
2759            .map(|metadata| ThreadTurnOverrides {
2760                tool_allowlist: metadata.tool_allowlist,
2761                developer_instructions: metadata.developer_instructions,
2762                external_tools: metadata.external_tools,
2763            })
2764            .unwrap_or_default())
2765    }
2766
2767    pub async fn set_thread_selection_mode(
2768        &self,
2769        thread_id: &ThreadId,
2770        selection_mode: ModelSelectionMode,
2771    ) -> anyhow::Result<()> {
2772        let Some(store) = &self.thread_store else {
2773            return Ok(());
2774        };
2775        let Some(snapshot) = store.load_thread(thread_id).await? else {
2776            anyhow::bail!("thread not found: {thread_id}");
2777        };
2778        let Some(mut metadata) = snapshot.metadata else {
2779            return Ok(());
2780        };
2781        let concrete = selection_mode.concrete_selection();
2782        metadata.provider = Some(concrete.provider);
2783        metadata.model = Some(concrete.model);
2784        metadata.selection_mode = Some(selection_mode);
2785        metadata.updated_at = OffsetDateTime::now_utc();
2786        store.update_thread_metadata(metadata).await?;
2787        Ok(())
2788    }
2789
2790    async fn runner_session_for_thread(
2791        &self,
2792        thread_id: &ThreadId,
2793    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2794        let metadata = if let Some(store) = &self.thread_store {
2795            store.load_thread_metadata(thread_id).await?
2796        } else {
2797            None
2798        };
2799        // An explicit per-thread binding wins over the runtime-level destination.
2800        let destination = metadata
2801            .as_ref()
2802            .and_then(|metadata| metadata.runner_binding.as_ref())
2803            .map(|binding| binding.destination.clone())
2804            .or(self.config.read().await.remote_runner_destination.clone());
2805        let Some(destination) = destination else {
2806            self.evict_runner_session(thread_id).await;
2807            return Ok(None);
2808        };
2809        let provider = self
2810            .registry
2811            .remote_runner_providers
2812            .iter()
2813            .find(|provider| provider.id() == destination.provider_id)
2814            .cloned()
2815            .ok_or_else(|| {
2816                anyhow::anyhow!(
2817                    "remote runner provider {:?} is not installed",
2818                    destination.provider_id
2819                )
2820            })?;
2821        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2822        let slot = {
2823            let mut sessions = self.runner_sessions.lock().await;
2824            match sessions.get(thread_id) {
2825                Some(slot) if slot.matches(&destination) => slot.clone(),
2826                _ => {
2827                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2828                    sessions.insert(thread_id.clone(), slot.clone());
2829                    slot
2830                }
2831            }
2832        };
2833
2834        loop {
2835            let initialized = slot.initialized.notified();
2836            let mut state = slot.state.lock().await;
2837            match &*state {
2838                RunnerSessionSlotState::Ready(cached) => {
2839                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2840                }
2841                RunnerSessionSlotState::Failed(error) => {
2842                    return Err(anyhow::anyhow!(error.to_string()));
2843                }
2844                RunnerSessionSlotState::Initializing => {
2845                    drop(state);
2846                    initialized.await;
2847                }
2848                RunnerSessionSlotState::Empty => {
2849                    *state = RunnerSessionSlotState::Initializing;
2850                    drop(state);
2851                    self.spawn_runner_session_initialization(
2852                        thread_id.clone(),
2853                        destination.clone(),
2854                        provider.clone(),
2855                        persisted_state.clone(),
2856                        slot.clone(),
2857                    );
2858                }
2859            }
2860        }
2861    }
2862
2863    fn spawn_runner_session_initialization(
2864        &self,
2865        thread_id: ThreadId,
2866        destination: RunnerDestination,
2867        provider: Arc<dyn RemoteRunnerProvider>,
2868        persisted_state: Option<RunnerSessionState>,
2869        slot: Arc<RunnerSessionSlot>,
2870    ) {
2871        let thread_store = self.thread_store.clone();
2872        let runner_sessions = self.runner_sessions.clone();
2873        // The task intentionally outlives the turn that first requested the
2874        // workspace. Interrupting that turn must not strand the slot in
2875        // `Initializing` and deadlock every later tool or lifecycle call.
2876        drop(tokio::spawn(async move {
2877            let initialized = async {
2878                let session = if let Some(state) = persisted_state
2879                    && state.provider_id == destination.provider_id
2880                    && state.destination_id == destination.id
2881                {
2882                    match provider.resume_session(state).await {
2883                        Ok(session) => session,
2884                        Err(_) => provider.create_session(destination.clone()).await?,
2885                    }
2886                } else {
2887                    provider.create_session(destination.clone()).await?
2888                };
2889                let resolved = (destination.clone(), session.clone());
2890                // Persist before releasing the singleflight or dispatching
2891                // the first tool. A later process can rejoin even if the turn
2892                // that requested initialization has already been interrupted.
2893                Self::persist_runner_state_in_store(
2894                    thread_store.as_ref(),
2895                    &thread_id,
2896                    Some(&resolved),
2897                )
2898                .await?;
2899                Ok::<_, anyhow::Error>(CachedRunnerSession {
2900                    destination,
2901                    session,
2902                })
2903            }
2904            .await;
2905
2906            match initialized {
2907                Ok(cached) => {
2908                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2909                    slot.initialized.notify_waiters();
2910                }
2911                Err(error) => {
2912                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2913                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2914                    slot.initialized.notify_waiters();
2915                    let mut sessions = runner_sessions.lock().await;
2916                    let is_current = sessions
2917                        .get(&thread_id)
2918                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2919                    if is_current {
2920                        sessions.remove(&thread_id);
2921                    }
2922                }
2923            }
2924        }));
2925    }
2926
2927    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2928        self.runner_sessions.lock().await.remove(thread_id);
2929    }
2930
2931    async fn cache_runner_session(
2932        &self,
2933        thread_id: &ThreadId,
2934        destination: RunnerDestination,
2935        session: Arc<dyn RemoteRunnerSession>,
2936    ) {
2937        let cached = CachedRunnerSession {
2938            destination,
2939            session,
2940        };
2941        self.runner_sessions.lock().await.insert(
2942            thread_id.clone(),
2943            Arc::new(RunnerSessionSlot::ready(cached)),
2944        );
2945    }
2946
2947    /// Read a thread's explicit runner binding without creating or resuming a
2948    /// session. Tool routing uses this before local previews so runner-backed
2949    /// and fail-closed hosted calls never inspect the host workspace.
2950    pub(crate) async fn runner_binding_for_thread(
2951        &self,
2952        thread_id: &ThreadId,
2953    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2954        let Some(store) = &self.thread_store else {
2955            return Ok(None);
2956        };
2957        Ok(store
2958            .load_thread_metadata(thread_id)
2959            .await?
2960            .and_then(|metadata| metadata.runner_binding))
2961    }
2962
2963    /// Resolve a previously-read binding into a live remote workspace. The
2964    /// session remains lazy: callers invoke this only after policy approval.
2965    pub(crate) async fn remote_workspace_for_binding(
2966        &self,
2967        thread_id: &ThreadId,
2968        binding: ThreadRunnerBinding,
2969    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2970        let session = self
2971            .runner_session_for_thread(thread_id)
2972            .await?
2973            .map(|(_, session)| session)
2974            .ok_or_else(|| {
2975                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2976            })?;
2977        Ok(Arc::new(RemoteWorkspace {
2978            session,
2979            root: binding.workspace,
2980            read_roots: binding.read_roots,
2981        }))
2982    }
2983
2984    pub(crate) async fn runner_tool_execution_lock(
2985        &self,
2986        session: &dyn RemoteRunnerSession,
2987    ) -> Arc<RunnerToolExecutionMutex> {
2988        let state = session.state();
2989        let key = (state.provider_id, state.session_id);
2990        let mut locks = self.runner_tool_execution_locks.lock().await;
2991        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2992            return lock;
2993        }
2994
2995        locks.retain(|_, lock| lock.strong_count() > 0);
2996        let lock = Arc::new(Mutex::new(()));
2997        locks.insert(key, Arc::downgrade(&lock));
2998        lock
2999    }
3000
3001    async fn persist_runner_state(
3002        &self,
3003        thread_id: &ThreadId,
3004        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3005    ) -> anyhow::Result<()> {
3006        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3007    }
3008
3009    async fn persist_runner_state_in_store(
3010        store: Option<&Arc<dyn ThreadStore>>,
3011        thread_id: &ThreadId,
3012        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3013    ) -> anyhow::Result<()> {
3014        let Some((destination, session)) = runner else {
3015            return Ok(());
3016        };
3017        let Some(store) = store else {
3018            return Ok(());
3019        };
3020        let Some(snapshot) = store.load_thread(thread_id).await? else {
3021            return Ok(());
3022        };
3023        let Some(mut metadata) = snapshot.metadata else {
3024            return Ok(());
3025        };
3026        metadata.runner_destination = Some(destination.clone());
3027        metadata.runner_state = Some(session.state());
3028        metadata.updated_at = OffsetDateTime::now_utc();
3029        store.update_thread_metadata(metadata).await?;
3030        Ok(())
3031    }
3032
3033    fn remote_runner_provider_by_id(
3034        &self,
3035        provider_id: &str,
3036    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3037        self.registry
3038            .remote_runner_providers
3039            .iter()
3040            .find(|provider| provider.id() == provider_id)
3041            .cloned()
3042    }
3043
3044    /// Resolve the live runner session for a thread along with its provider,
3045    /// failing clearly when the thread is not runner-bound.
3046    async fn thread_runner_session(
3047        &self,
3048        thread_id: &ThreadId,
3049    ) -> anyhow::Result<(
3050        RunnerDestination,
3051        Arc<dyn RemoteRunnerProvider>,
3052        Arc<dyn RemoteRunnerSession>,
3053    )> {
3054        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3055            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3056        };
3057        let provider = self
3058            .remote_runner_provider_by_id(&destination.provider_id)
3059            .ok_or_else(|| {
3060                anyhow::anyhow!(
3061                    "remote runner provider {:?} is not installed",
3062                    destination.provider_id
3063                )
3064            })?;
3065        Ok((destination, provider, session))
3066    }
3067
3068    /// Pause a runner-bound thread's session toward standby and persist the
3069    /// post-pause state. Errors if the provider is not pausable.
3070    pub async fn pause_thread_runner(
3071        &self,
3072        thread_id: &ThreadId,
3073    ) -> anyhow::Result<RunnerSessionState> {
3074        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3075        anyhow::ensure!(
3076            provider.capabilities().pausable,
3077            "remote runner provider {:?} does not support pausing",
3078            destination.provider_id
3079        );
3080        let state = session.pause().await?;
3081        self.persist_runner_state(thread_id, Some(&(destination, session)))
3082            .await?;
3083        Ok(state)
3084    }
3085
3086    /// Resume (wake) a runner-bound thread's paused session and persist state.
3087    pub async fn resume_thread_runner(
3088        &self,
3089        thread_id: &ThreadId,
3090    ) -> anyhow::Result<RunnerSessionState> {
3091        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3092        let state = session.resume().await?;
3093        self.persist_runner_state(thread_id, Some(&(destination, session)))
3094            .await?;
3095        Ok(state)
3096    }
3097
3098    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3099    /// state and leave the remote sandbox alive. Errors if not detachable.
3100    pub async fn detach_thread_runner(
3101        &self,
3102        thread_id: &ThreadId,
3103    ) -> anyhow::Result<RunnerSessionState> {
3104        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3105        anyhow::ensure!(
3106            provider.capabilities().detachable,
3107            "remote runner provider {:?} does not support detaching",
3108            destination.provider_id
3109        );
3110        let state = session.detach().await?;
3111        // Persist the detached state explicitly so a later turn or process
3112        // rejoins the same sandbox instead of provisioning a new one.
3113        if let Some(store) = &self.thread_store
3114            && let Some(snapshot) = store.load_thread(thread_id).await?
3115            && let Some(mut metadata) = snapshot.metadata
3116        {
3117            metadata.runner_destination = Some(destination);
3118            metadata.runner_state = Some(state.clone());
3119            metadata.updated_at = OffsetDateTime::now_utc();
3120            store.update_thread_metadata(metadata).await?;
3121        }
3122        self.evict_runner_session(thread_id).await;
3123        Ok(state)
3124    }
3125
3126    /// Rejoin a previously created sandbox from a thread's persisted runner
3127    /// state without provisioning a new one. An optional `sandbox` overrides the
3128    /// persisted sandbox name (recovery by name). Persists refreshed state.
3129    pub async fn rejoin_thread_runner(
3130        &self,
3131        thread_id: &ThreadId,
3132        sandbox: Option<String>,
3133    ) -> anyhow::Result<RunnerSessionState> {
3134        let store = self
3135            .thread_store
3136            .as_ref()
3137            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3138        let metadata = store
3139            .load_thread_metadata(thread_id)
3140            .await?
3141            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3142        let destination = metadata
3143            .runner_binding
3144            .as_ref()
3145            .map(|binding| binding.destination.clone())
3146            .or_else(|| metadata.runner_destination.clone())
3147            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3148        let mut state = metadata
3149            .runner_state
3150            .clone()
3151            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3152        if let Some(sandbox) = sandbox
3153            && let Some(object) = state.metadata.as_object_mut()
3154        {
3155            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3156        }
3157        let provider = self
3158            .remote_runner_provider_by_id(&destination.provider_id)
3159            .ok_or_else(|| {
3160                anyhow::anyhow!(
3161                    "remote runner provider {:?} is not installed",
3162                    destination.provider_id
3163                )
3164            })?;
3165        let session = provider.rejoin_session(state).await?;
3166        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3167            .await?;
3168        self.cache_runner_session(thread_id, destination, session.clone())
3169            .await;
3170        Ok(session.state())
3171    }
3172
3173    async fn record_thread_usage_metadata(
3174        &self,
3175        thread_id: &ThreadId,
3176        usage: &TokenUsage,
3177    ) -> anyhow::Result<()> {
3178        if usage.is_empty() {
3179            return Ok(());
3180        }
3181        let Some(store) = &self.thread_store else {
3182            return Ok(());
3183        };
3184        let Some(snapshot) = store.load_thread(thread_id).await? else {
3185            return Ok(());
3186        };
3187        let Some(mut metadata) = snapshot.metadata else {
3188            return Ok(());
3189        };
3190        metadata
3191            .usage
3192            .get_or_insert_with(ThreadUsageMetadata::default)
3193            .add_token_usage(usage);
3194        metadata.updated_at = OffsetDateTime::now_utc();
3195        store.update_thread_metadata(metadata).await?;
3196        Ok(())
3197    }
3198
3199    pub fn start_turn(
3200        self: &Arc<Self>,
3201        mut req: StartTurnRequest,
3202    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3203        Box::pin(async move {
3204            let turn_admission = self.turn_admission.lock().await;
3205            anyhow::ensure!(
3206                self.accepting_turns.load(Ordering::Acquire),
3207                "runtime is quiescing and cannot accept new turns"
3208            );
3209            req.workspace = validate_thread_workspace(&req.workspace)?;
3210            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3211            let cfg = self.config.read().await.clone();
3212            let provider = req
3213                .provider_override
3214                .clone()
3215                .unwrap_or_else(|| cfg.default_provider.clone());
3216            self.engine_for(&provider)?;
3217            let turn_id = uuid::Uuid::new_v4().to_string();
3218            let mut initial_mailbox_ack = None;
3219            if let Some((team_id, member)) = &team_member {
3220                let pending = self
3221                    .teams
3222                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3223                    .await?;
3224                if !pending.is_empty()
3225                    && let Some(team) = self.read_team(team_id).await
3226                {
3227                    let mailbox = format_mailbox_messages(&team, &pending);
3228                    req.message = if req.message.trim().is_empty() {
3229                        mailbox
3230                    } else {
3231                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3232                    };
3233                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3234                        team_id: team_id.clone(),
3235                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3236                    });
3237                }
3238            }
3239            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3240            let drain = Arc::new(TurnDrainHandle {
3241                thread_id: req.thread_id.clone(),
3242                interrupt_requested: AtomicBool::new(false),
3243                interrupt_reason: Mutex::new(None),
3244                completed: AtomicBool::new(false),
3245                completed_notify: Notify::new(),
3246            });
3247            let active = ActiveTurnHandle {
3248                thread_id: req.thread_id.clone(),
3249                abort: abort_handle,
3250                steers: Arc::new(Mutex::new(Vec::new())),
3251                drain,
3252            };
3253            self.active_turns
3254                .write()
3255                .await
3256                .insert(turn_id.clone(), active);
3257            self.record_turn_lifecycle(
3258                req.thread_id.clone(),
3259                turn_id.clone(),
3260                TurnLifecycleState::Running,
3261                TurnCleanupState::NotRequested,
3262                None,
3263            )
3264            .await;
3265            self.active_turn_contexts.write().await.insert(
3266                turn_id.clone(),
3267                InheritedTurnContext {
3268                    workspace: req.workspace.clone(),
3269                    instructions: req.instructions.clone(),
3270                    developer_context: req.developer_context.clone(),
3271                },
3272            );
3273            if let Some((team_id, member)) = team_member {
3274                let updated = match self
3275                    .teams
3276                    .update_member(&team_id, &member.id, |member| {
3277                        member.current_turn_id = Some(turn_id.clone());
3278                        member.status = TeamMemberStatus::Running;
3279                        member.final_message = None;
3280                        member.terminal_error = None;
3281                    })
3282                    .await
3283                {
3284                    Ok(updated) => updated,
3285                    Err(error) => {
3286                        self.active_turns.write().await.remove(&turn_id);
3287                        self.active_turn_contexts.write().await.remove(&turn_id);
3288                        self.teams
3289                            .release_mailbox_reservations_for_turn(&turn_id)
3290                            .await;
3291                        return Err(error);
3292                    }
3293                };
3294                if let Some(member) = updated
3295                    .members
3296                    .into_iter()
3297                    .find(|candidate| candidate.id == member.id)
3298                {
3299                    self.emit(RoderEvent::TeamMemberStatusChanged(
3300                        TeamMemberStatusChanged {
3301                            team_id,
3302                            member_id: member.id,
3303                            member_thread_id: member.thread_id,
3304                            status: TeamMemberStatus::Running,
3305                            timestamp: OffsetDateTime::now_utc(),
3306                        },
3307                    ))
3308                    .await;
3309                }
3310            }
3311            let runtime = Arc::clone(self);
3312            let turn_req = req;
3313            let thread_id_for_task = turn_req.thread_id.clone();
3314            let turn_id_for_task = turn_id.clone();
3315            tokio::spawn(async move {
3316                let result = Abortable::new(
3317                    runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3318                    abort_registration,
3319                )
3320                .await;
3321                /*
3322                 * A failed sibling in a parallel tool batch drops in-flight external tool
3323                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3324                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3325                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3326                 * resolution; on clean completion the map holds nothing for this turn.
3327                 */
3328                runtime
3329                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3330                    .await;
3331                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3332                match &result {
3333                    Ok(Err(err)) => {
3334                        let (cleanup, ownership) =
3335                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3336                        // run_turn emits failures after the stream starts; this covers setup/startup errors.
3337                        runtime
3338                            .emit(RoderEvent::TurnFailed(TurnFailed {
3339                                thread_id: thread_id_for_task.clone(),
3340                                turn_id: turn_id_for_task.clone(),
3341                                error: err.to_string(),
3342                                error_kind: None,
3343                                usage: None,
3344                                timestamp: OffsetDateTime::now_utc(),
3345                            }))
3346                            .await;
3347                        runtime
3348                            .record_turn_lifecycle_with_ownership(
3349                                thread_id_for_task.clone(),
3350                                turn_id_for_task.clone(),
3351                                TurnLifecycleState::Failed,
3352                                cleanup,
3353                                Some(TurnLifecycleReason::ProviderFailure),
3354                                ownership,
3355                            )
3356                            .await;
3357                        let _ = runtime
3358                            .complete_team_member_turn_with_result(
3359                                &thread_id_for_task,
3360                                &turn_id_for_task,
3361                                TeamMemberStatus::Failed,
3362                                None,
3363                                Some(err.to_string()),
3364                            )
3365                            .await;
3366                    }
3367                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3368                        let (cleanup, ownership) =
3369                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3370                        runtime
3371                            .record_turn_lifecycle_with_ownership(
3372                                thread_id_for_task.clone(),
3373                                turn_id_for_task.clone(),
3374                                TurnLifecycleState::Failed,
3375                                cleanup,
3376                                Some(TurnLifecycleReason::ProviderFailure),
3377                                ownership,
3378                            )
3379                            .await;
3380                        let _ = runtime
3381                            .complete_team_member_turn_with_result(
3382                                &thread_id_for_task,
3383                                &turn_id_for_task,
3384                                TeamMemberStatus::Failed,
3385                                None,
3386                                Some("turn stopped before completion".to_string()),
3387                            )
3388                            .await;
3389                    }
3390                    Err(_) => {
3391                        let reason = if let Some(handle) = runtime
3392                            .turn_drains
3393                            .read()
3394                            .await
3395                            .get(&turn_id_for_task)
3396                            .cloned()
3397                        {
3398                            handle
3399                                .interrupt_reason
3400                                .lock()
3401                                .await
3402                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3403                        } else {
3404                            TurnLifecycleReason::RuntimeFailure
3405                        };
3406                        let (cleanup, ownership) =
3407                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3408                        runtime
3409                            .record_turn_lifecycle_with_ownership(
3410                                thread_id_for_task.clone(),
3411                                turn_id_for_task.clone(),
3412                                TurnLifecycleState::Interrupted,
3413                                cleanup,
3414                                Some(reason),
3415                                ownership,
3416                            )
3417                            .await;
3418                        runtime
3419                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3420                                thread_id: thread_id_for_task.clone(),
3421                                turn_id: turn_id_for_task.clone(),
3422                                timestamp: OffsetDateTime::now_utc(),
3423                            }))
3424                            .await;
3425                        let _ = runtime
3426                            .complete_team_member_turn_with_result(
3427                                &thread_id_for_task,
3428                                &turn_id_for_task,
3429                                TeamMemberStatus::Interrupted,
3430                                None,
3431                                None,
3432                            )
3433                            .await;
3434                    }
3435                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3436                }
3437                runtime
3438                    .teams
3439                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3440                    .await;
3441                runtime.active_turns.write().await.remove(&turn_id_for_task);
3442                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3443                    drain.completed.store(true, Ordering::Release);
3444                    drain.completed_notify.notify_waiters();
3445                }
3446                runtime
3447                    .active_turn_selections
3448                    .write()
3449                    .await
3450                    .remove(&turn_id_for_task);
3451                runtime
3452                    .active_turn_contexts
3453                    .write()
3454                    .await
3455                    .remove(&turn_id_for_task);
3456                if !completed {
3457                    // Completion paths above consume registered provider cleanup
3458                    // handles. A setup failure before an engine can stream has no
3459                    // such handle; this is a harmless final defensive sweep.
3460                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3461                        cleanups.remove(&turn_id_for_task);
3462                    });
3463                }
3464                runtime.active_turns_changed.notify_waiters();
3465                if completed {
3466                    let _ = runtime
3467                        .continue_active_goal_after_turn(thread_id_for_task)
3468                        .await;
3469                }
3470            });
3471            drop(turn_admission);
3472            Ok(turn_id)
3473        })
3474    }
3475
3476    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3477        self.active_turns
3478            .read()
3479            .await
3480            .values()
3481            .any(|handle| &handle.thread_id == thread_id)
3482    }
3483
3484    async fn ensure_codex_v2_team_capacity(
3485        &self,
3486        team: &TeamState,
3487        resuming_member_id: &str,
3488        candidate_model: Option<&str>,
3489    ) -> anyhow::Result<()> {
3490        if !is_codex_v2_team(team)
3491            && !candidate_model
3492                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3493        {
3494            return Ok(());
3495        }
3496        let active_thread_ids = self
3497            .active_turns
3498            .read()
3499            .await
3500            .values()
3501            .map(|handle| handle.thread_id.clone())
3502            .collect::<std::collections::HashSet<_>>();
3503        let resident_threads = 1 + team
3504            .members
3505            .iter()
3506            .filter(|member| {
3507                member.role != roder_api::teams::TeamMemberRole::Lead
3508                    && member.id != resuming_member_id
3509                    && active_thread_ids.contains(&member.thread_id)
3510            })
3511            .count();
3512        anyhow::ensure!(
3513            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3514            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3515            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3516        );
3517        Ok(())
3518    }
3519
3520    /// Number of currently running turns across all threads. Hosted runtime
3521    /// pools use this to avoid evicting tenants with active work.
3522    pub async fn active_turn_count(&self) -> usize {
3523        self.active_turns.read().await.len()
3524    }
3525
3526    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3527        self.active_turns
3528            .read()
3529            .await
3530            .iter()
3531            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3532    }
3533
3534    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3535        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3536        let draining_turn_id = if active_turn_id.is_none() {
3537            self.turn_drains
3538                .read()
3539                .await
3540                .iter()
3541                .find_map(|(turn_id, drain)| {
3542                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3543                })
3544        } else {
3545            None
3546        };
3547        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3548            return ThreadActivity::default();
3549        };
3550
3551        let mut active_flags = Vec::new();
3552        {
3553            let pending_approvals = self.pending_tool_approvals.lock().await;
3554            if pending_approvals
3555                .values()
3556                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3557            {
3558                active_flags.push("approvalRequired".to_string());
3559            }
3560        }
3561        {
3562            let pending_inputs = self.pending_user_inputs.lock().await;
3563            if pending_inputs
3564                .values()
3565                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3566            {
3567                active_flags.push("userInputRequired".to_string());
3568            }
3569        }
3570        {
3571            let pending_external = self.pending_external_tool_calls.lock().await;
3572            if pending_external
3573                .values()
3574                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3575            {
3576                active_flags.push("externalToolPending".to_string());
3577            }
3578        }
3579        let interrupting = self
3580            .active_turns
3581            .read()
3582            .await
3583            .get(&active_turn_id)
3584            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3585            || self
3586                .turn_drains
3587                .read()
3588                .await
3589                .get(&active_turn_id)
3590                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3591        if interrupting {
3592            active_flags.push("interrupting".to_string());
3593        }
3594        if self.pending_plan_exit().await.is_some_and(|pending| {
3595            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3596        }) {
3597            active_flags.push("planExitRequired".to_string());
3598        }
3599
3600        ThreadActivity {
3601            active_turn_id: Some(active_turn_id),
3602            active_flags,
3603        }
3604    }
3605
3606    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3607        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3608            .await
3609    }
3610
3611    async fn interrupt_turn_with_reason(
3612        &self,
3613        thread_id: ThreadId,
3614        turn_id: TurnId,
3615        reason: TurnLifecycleReason,
3616    ) -> anyhow::Result<()> {
3617        let handle = self.active_turns.write().await.remove(&turn_id);
3618        if let Some(handle) = handle {
3619            if handle
3620                .drain
3621                .interrupt_requested
3622                .swap(true, Ordering::AcqRel)
3623            {
3624                return Ok(());
3625            }
3626            *handle.drain.interrupt_reason.lock().await = Some(reason);
3627            self.turn_drains
3628                .write()
3629                .await
3630                .insert(turn_id.clone(), handle.drain.clone());
3631            let ownership = self.provider_cleanup_ownership(&turn_id);
3632            self.record_turn_lifecycle_with_ownership(
3633                thread_id.clone(),
3634                turn_id.clone(),
3635                TurnLifecycleState::InterruptRequested,
3636                TurnCleanupState::Requested,
3637                Some(reason),
3638                ownership,
3639            )
3640            .await;
3641            handle.abort.abort();
3642            self.active_turns_changed.notify_waiters();
3643        }
3644        self.active_turn_selections.write().await.remove(&turn_id);
3645        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3646            .await;
3647        Ok(())
3648    }
3649
3650    pub async fn steer_turn(
3651        &self,
3652        thread_id: ThreadId,
3653        turn_id: TurnId,
3654        message: String,
3655        images: Vec<InputImage>,
3656    ) -> anyhow::Result<()> {
3657        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3658            .await
3659    }
3660
3661    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3662        let active = self.active_turns.read().await.get(turn_id).cloned();
3663        let Some(active) = active else {
3664            return false;
3665        };
3666        let has_pending = !active.steers.lock().await.is_empty();
3667        has_pending
3668    }
3669
3670    async fn steer_turn_with_mailbox_ack(
3671        &self,
3672        thread_id: ThreadId,
3673        turn_id: TurnId,
3674        message: String,
3675        message_ids: Vec<String>,
3676        team_id: TeamId,
3677    ) -> anyhow::Result<()> {
3678        self.enqueue_turn_steer(
3679            thread_id,
3680            turn_id,
3681            message,
3682            Vec::new(),
3683            Some(MailboxDeliveryAck {
3684                team_id,
3685                message_ids,
3686            }),
3687        )
3688        .await
3689    }
3690
3691    async fn deliver_pending_team_mailbox(
3692        &self,
3693        team_id: &str,
3694        member_id: &str,
3695        member_thread_id: ThreadId,
3696        turn_id: TurnId,
3697    ) -> anyhow::Result<()> {
3698        let pending = self
3699            .teams
3700            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3701            .await?;
3702        if pending.is_empty() {
3703            return Ok(());
3704        }
3705        let message_ids = pending
3706            .iter()
3707            .map(|message| message.id.clone())
3708            .collect::<Vec<_>>();
3709        let team = self
3710            .read_team(team_id)
3711            .await
3712            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3713        if let Err(error) = self
3714            .steer_turn_with_mailbox_ack(
3715                member_thread_id,
3716                turn_id.clone(),
3717                format_mailbox_messages(&team, &pending),
3718                message_ids.clone(),
3719                team_id.to_string(),
3720            )
3721            .await
3722        {
3723            self.teams
3724                .release_mailbox_reservations(&turn_id, &message_ids)
3725                .await;
3726            return Err(error);
3727        }
3728        Ok(())
3729    }
3730
3731    async fn enqueue_turn_steer(
3732        &self,
3733        thread_id: ThreadId,
3734        turn_id: TurnId,
3735        message: String,
3736        images: Vec<InputImage>,
3737        mailbox_ack: Option<MailboxDeliveryAck>,
3738    ) -> anyhow::Result<()> {
3739        let message = message.trim().to_string();
3740        if message.is_empty() && images.is_empty() {
3741            return Ok(());
3742        }
3743
3744        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3745            anyhow::bail!("no active turn to steer");
3746        };
3747        active.steers.lock().await.push(QueuedTurnSteer {
3748            message: UserMessage::with_images(message.clone(), images),
3749            mailbox_ack,
3750        });
3751        self.emit(RoderEvent::TurnSteered(TurnSteered {
3752            thread_id,
3753            turn_id,
3754            message,
3755            timestamp: OffsetDateTime::now_utc(),
3756        }))
3757        .await;
3758        Ok(())
3759    }
3760
3761    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3762        let cfg = self.config.read().await;
3763        let model_profile =
3764            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3765        self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3766    }
3767
3768    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3769        self.registry
3770            .subagent_dispatchers
3771            .iter()
3772            .flat_map(|dispatcher| dispatcher.definitions())
3773            .collect()
3774    }
3775
3776    async fn run_turn(
3777        self: &Arc<Self>,
3778        req: StartTurnRequest,
3779        turn_id: TurnId,
3780        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3781    ) -> anyhow::Result<TurnRunOutcome> {
3782        let turn_started_at = OffsetDateTime::now_utc();
3783        self.emit(RoderEvent::TurnStarted(TurnStarted {
3784            thread_id: req.thread_id.clone(),
3785            turn_id: turn_id.clone(),
3786            runtime_profile: self.config.read().await.runtime_profile,
3787            timestamp: turn_started_at,
3788        }))
3789        .await;
3790        self.persist_turn_item(
3791            &req.thread_id,
3792            &turn_id,
3793            &TranscriptItem::UserMessage(UserMessage::with_images(
3794                req.message.clone(),
3795                req.images.clone(),
3796            )),
3797        )
3798        .await?;
3799        crate::hooks::run_lifecycle(
3800            self,
3801            &req.thread_id,
3802            &turn_id,
3803            Some(&req.workspace),
3804            "UserPromptSubmit",
3805            None,
3806            serde_json::json!({"prompt": req.message}),
3807        )
3808        .await;
3809        if let Some(ack) = initial_mailbox_ack {
3810            self.teams
3811                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3812                .await?;
3813        }
3814
3815        let mut cfg = self.config.read().await.clone();
3816        let runtime_profile = cfg.runtime_profile;
3817        let turn_deadline = turn_deadline_for_config(&cfg);
3818        let deadline_finalization_reserve =
3819            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3820        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3821        let concrete_selection = selection_mode
3822            .as_ref()
3823            .map(ModelSelectionMode::concrete_selection);
3824        let default_provider = req
3825            .provider_override
3826            .clone()
3827            .or_else(|| {
3828                concrete_selection
3829                    .as_ref()
3830                    .map(|selection| selection.provider.clone())
3831            })
3832            .unwrap_or(cfg.default_provider.clone());
3833        let default_model = req
3834            .model_override
3835            .clone()
3836            .or_else(|| {
3837                concrete_selection
3838                    .as_ref()
3839                    .map(|selection| selection.model.clone())
3840            })
3841            .unwrap_or(cfg.default_model.clone());
3842        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3843            selection_mode
3844                .as_ref()
3845                .and_then(ModelSelectionMode::reasoning)
3846        }) {
3847            validate_reasoning_effort(&default_model, reasoning)?;
3848            cfg.reasoning = Some(reasoning.to_string());
3849        }
3850        let turn_has_concrete_model_override =
3851            req.provider_override.is_some() || req.model_override.is_some();
3852        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3853            Some(ModelSelectionMode::Auto {
3854                router_id, profile, ..
3855            }) if !turn_has_concrete_model_override => (
3856                RuntimeInferenceRouterConfig {
3857                    enabled: true,
3858                    router_id: Some(router_id.clone()),
3859                },
3860                profile.clone(),
3861            ),
3862            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3863        };
3864        let mut provider = default_provider.clone();
3865        let mut model = default_model.clone();
3866        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3867        let workspace = req.workspace.clone();
3868        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3869        let mut compacted_this_turn = transcript
3870            .iter()
3871            .any(crate::compaction::is_compaction_boundary);
3872        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3873        let agent_swarm_mode_active = self
3874            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3875            .await;
3876        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3877        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3878        let mut final_assistant_text = String::new();
3879        let mut final_phase_messages = Vec::<AssistantMessage>::new();
3880        let mut final_reasoning_text = String::new();
3881        let mut final_provider_metadata = None;
3882        let mut exhausted_tool_rounds = true;
3883        let mut verification_gate =
3884            VerificationGateState::new(req.message.clone(), runtime_profile);
3885        let mut speed_policy = SpeedPolicyState::default();
3886        let mut reliability = TurnReliabilityState::default();
3887        let mut turn_usage = TokenUsage::default();
3888        // Overwritten on every inference step's Completed event so only the
3889        // terminal step's stop reason survives (mid-turn tool_use steps must
3890        // not leak onto turn/completed).
3891        let mut turn_finish_reason: Option<String> = None;
3892        let mut deadline_finalization_requested = false;
3893        let mut deadline_scoreable_completion_requested = false;
3894        let mut task_ledger_completion_reminders = 0_u8;
3895        let mut task_ledger_scoreable_checkpoints = 0_u8;
3896        let mut empty_tool_call_nudges_used = 0_u32;
3897        let mut provider_stream_retry_attempts = 0_u32;
3898        let mut context_limit_recovery_attempts = 0_u32;
3899        let mut routing_candidates = None;
3900        let routing_transcript_start = transcript.len().saturating_sub(1);
3901        let mut routing_escalations = 0_u32;
3902        let mut model_switch_summary_selection = None::<ModelSelection>;
3903
3904        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3905            if let Some(deadline) = turn_deadline
3906                && deadline_expired(deadline)
3907            {
3908                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3909                    .await?;
3910                return Ok(TurnRunOutcome::Stopped);
3911            }
3912            let steers = self.drain_turn_steers(&turn_id).await;
3913            self.append_steers(&req, &turn_id, &mut transcript, steers)
3914                .await?;
3915            if runtime_profile == RuntimeProfile::Eval
3916                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3917                    turn_deadline,
3918                    deadline_finalization_reserve,
3919                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3920                )
3921            {
3922                if req.task_ledger_required
3923                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3924                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3925                {
3926                    task_ledger_completion_reminders += 1;
3927                    deadline_scoreable_completion_requested = true;
3928                    let item = TranscriptItem::UserMessage(UserMessage::text(
3929                        task_ledger_deadline_completion_prompt(
3930                            remaining,
3931                            deadline_finalization_reserve,
3932                            &prompt,
3933                        ),
3934                    ));
3935                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3936                        .await?;
3937                    transcript.push(item);
3938                    continue 'tool_rounds;
3939                } else {
3940                    self.start_deadline_finalization(
3941                        &req.thread_id,
3942                        &turn_id,
3943                        &mut transcript,
3944                        remaining,
3945                    )
3946                    .await?;
3947                    deadline_finalization_requested = true;
3948                }
3949            }
3950            if runtime_profile == RuntimeProfile::Eval
3951                && req.task_ledger_required
3952                && !deadline_finalization_requested
3953                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3954                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3955                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3956                && remaining > deadline_finalization_reserve
3957                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3958            {
3959                task_ledger_scoreable_checkpoints += 1;
3960                let item = TranscriptItem::UserMessage(UserMessage::text(
3961                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3962                ));
3963                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3964                    .await?;
3965                transcript.push(item);
3966                continue 'tool_rounds;
3967            }
3968            if turn_inference_router.is_active() && routing_candidates.is_none() {
3969                routing_candidates =
3970                    Some(collect_inference_routing_candidates(&self.registry).await);
3971            }
3972            let routing_tools_model = model.clone();
3973            let routing_tools = self.filtered_tool_specs(
3974                &cfg,
3975                &model,
3976                model_profile.as_ref(),
3977                &thread_overrides.tool_allowlist,
3978                &thread_overrides.external_tools,
3979            );
3980            let prior_failures =
3981                transcript_failure_count_since(&transcript, routing_transcript_start)
3982                    .max(reliability.tool_failure_count())
3983                    .saturating_add(provider_stream_retry_attempts);
3984            let routing_selection = route_inference_selection(
3985                &self.registry,
3986                &turn_inference_router,
3987                InferenceRoutingRequest {
3988                    thread_id: &req.thread_id,
3989                    turn_id: &turn_id,
3990                    round_index: round_index as u32,
3991                    runtime_profile,
3992                    phase: speed_policy.phase(),
3993                    profile: turn_inference_router_profile.as_deref(),
3994                    default_selection: ModelSelection {
3995                        provider: default_provider.clone(),
3996                        model: default_model.clone(),
3997                    },
3998                    transcript: &transcript,
3999                    tools: &routing_tools,
4000                    candidates: routing_candidates.as_deref(),
4001                    prior_failures,
4002                    prior_escalations: routing_escalations,
4003                },
4004            )
4005            .await;
4006            if let Some(decision) = routing_selection.decision.clone() {
4007                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4008                    routing_escalations = routing_escalations.saturating_add(1);
4009                }
4010                self.emit(RoderEvent::InferenceRoutingDecision(
4011                    InferenceRoutingDecisionEvent {
4012                        thread_id: req.thread_id.clone(),
4013                        turn_id: turn_id.clone(),
4014                        round_index: round_index as u32,
4015                        default_selection: ModelSelection {
4016                            provider: default_provider.clone(),
4017                            model: default_model.clone(),
4018                        },
4019                        selected_selection: routing_selection.selection.clone(),
4020                        decision,
4021                        timestamp: OffsetDateTime::now_utc(),
4022                    },
4023                ))
4024                .await;
4025            }
4026            provider = routing_selection.selection.provider.clone();
4027            model = routing_selection.selection.model.clone();
4028            let engine = self.engine_for(&provider)?;
4029            let capabilities = engine.capabilities();
4030            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4031            let tools = if capabilities.tool_calls {
4032                if model == routing_tools_model {
4033                    routing_tools.clone()
4034                } else {
4035                    self.filtered_tool_specs(
4036                        &cfg,
4037                        &model,
4038                        model_profile.as_ref(),
4039                        &thread_overrides.tool_allowlist,
4040                        &thread_overrides.external_tools,
4041                    )
4042                }
4043            } else {
4044                Vec::new()
4045            };
4046            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4047            let tool_choice = if tools.is_empty() {
4048                ToolChoice::None
4049            } else {
4050                ToolChoice::Auto
4051            };
4052            let summary_selection = ModelSelection {
4053                provider: provider.clone(),
4054                model: model.clone(),
4055            };
4056            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4057                if let Some(summary) = model_switch_summary(
4058                    &transcript,
4059                    model_profile.as_ref(),
4060                    &provider,
4061                    &model,
4062                    &tools,
4063                ) {
4064                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4065                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4066                        .await?;
4067                    transcript.push(item);
4068                }
4069                model_switch_summary_selection = Some(summary_selection);
4070            }
4071
4072            if !capabilities.image_input && transcript_has_images(&transcript) {
4073                self.fail_turn_with_error(
4074                    &req.thread_id,
4075                    &turn_id,
4076                    format!("provider {provider} does not support image input"),
4077                )
4078                .await?;
4079                return Ok(TurnRunOutcome::Stopped);
4080            }
4081            transcript = self
4082                .compact_transcript_if_needed(
4083                    &req.thread_id,
4084                    &turn_id,
4085                    &provider,
4086                    &model,
4087                    transcript,
4088                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4089                )
4090                .await?;
4091            compacted_this_turn = compacted_this_turn
4092                || transcript
4093                    .iter()
4094                    .any(crate::compaction::is_compaction_boundary);
4095
4096            let speed_policy_decision =
4097                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4098            let request_reasoning = reasoning_from_decision(
4099                speed_policy_decision.as_ref(),
4100                routing_selection
4101                    .reasoning
4102                    .clone()
4103                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4104            );
4105            self.active_turn_selections.write().await.insert(
4106                turn_id.clone(),
4107                ModelSelectionMode::manual(
4108                    provider.clone(),
4109                    model.clone(),
4110                    request_reasoning.level.clone(),
4111                ),
4112            );
4113            if let Some(limit) = reliability.record_model_call(
4114                &cfg.reliability,
4115                runtime_profile == RuntimeProfile::Interactive,
4116            ) {
4117                self.fail_turn_due_to_reliability_limit(
4118                    &req.thread_id,
4119                    &turn_id,
4120                    &provider,
4121                    &model,
4122                    limit,
4123                    &transcript,
4124                )
4125                .await?;
4126                return Ok(TurnRunOutcome::Stopped);
4127            }
4128            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4129                thread_id: req.thread_id.clone(),
4130                turn_id: turn_id.clone(),
4131                engine_id: engine.id(),
4132                model: ModelSelection {
4133                    provider: provider.clone(),
4134                    model: model.clone(),
4135                },
4136                reasoning: request_reasoning.clone(),
4137                speed_policy: speed_policy_decision.clone(),
4138                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4139                timestamp: OffsetDateTime::now_utc(),
4140            }))
4141            .await;
4142
4143            let mut instructions = req.instructions.clone();
4144            if let Some(extra) = &thread_overrides.developer_instructions {
4145                instructions = apply_thread_developer_instructions(instructions, extra);
4146            }
4147            if let Some(context) = req.developer_context.as_deref() {
4148                instructions = apply_turn_developer_context(instructions, context);
4149            }
4150            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4151            if let Some(profile) = &model_profile {
4152                instructions = apply_model_instruction_overlay(instructions, profile);
4153            }
4154            if req.task_ledger_required
4155                && runtime_profile == RuntimeProfile::Eval
4156                && !transcript_has_task_ledger(&transcript)
4157            {
4158                instructions = apply_task_ledger_required(instructions);
4159            }
4160            if effective_policy_mode == PolicyMode::Plan {
4161                instructions = apply_plan_mode(instructions);
4162            }
4163            if agent_swarm_mode_active {
4164                instructions = apply_agent_swarm_mode(instructions);
4165            }
4166            // Ultra mode (any model) enables proactive multi-agent policy.
4167            // Codex Sol/Terra Ultra effort still does too without requiring the
4168            // mode flag. Models that advertise Ultra effort keep the
4169            // explicit-request-only policy on lower efforts when ultra mode is
4170            // off; other models get multi-agent policy only when ultra mode is on.
4171            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4172            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4173            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4174            if ultra_mode_active || model_has_ultra_effort {
4175                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4176            }
4177            instructions = self
4178                .goals
4179                .apply_goal_instructions(&req.thread_id, instructions)
4180                .await?;
4181            instructions =
4182                apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4183            let mut request_metadata = serde_json::json!({});
4184            if let Some(decision) = &speed_policy_decision {
4185                request_metadata["speedPolicy"] = serde_json::json!(decision);
4186            }
4187            if let Some(decision) = routing_selection.decision.as_ref() {
4188                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4189            }
4190            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4191                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4192            }
4193            if let Some(profile) = &model_profile {
4194                request_metadata["modelProfile"] = serde_json::json!({
4195                    "model": profile.model,
4196                    "providerFamily": profile.provider_family,
4197                    "editTool": profile.edit_tool,
4198                    "schemaPolicy": profile.schema_policy,
4199                    "instructionOverlay": profile.instruction_overlay,
4200                    "parallelToolCalls": profile.parallel_tool_calls,
4201                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4202                });
4203            }
4204            let task_ledger_required_this_round = req.task_ledger_required
4205                && runtime_profile == RuntimeProfile::Eval
4206                && !deadline_finalization_requested
4207                && !transcript_has_task_ledger(&transcript);
4208            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4209                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4210                .filter(|tools| !tools.is_empty());
4211            let request_tools = if deadline_finalization_requested {
4212                Vec::new()
4213            } else if let Some(ledger_tools) = &task_ledger_tools {
4214                ledger_tools.clone()
4215            } else {
4216                tools.clone()
4217            };
4218            let request_tool_choice = if deadline_finalization_requested {
4219                ToolChoice::None
4220            } else if task_ledger_tools.is_some() {
4221                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4222            } else {
4223                tool_choice.clone()
4224            };
4225            if deadline_finalization_requested {
4226                request_metadata["deadlineFinalization"] = serde_json::json!({
4227                    "reserveSeconds": deadline_finalization_reserve,
4228                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4229                });
4230            } else if deadline_scoreable_completion_requested {
4231                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4232                    "reserveSeconds": deadline_finalization_reserve,
4233                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4234                });
4235            }
4236            let request = AgentInferenceRequest {
4237                model: ModelSelection {
4238                    provider: provider.clone(),
4239                    model: model.clone(),
4240                },
4241                instructions,
4242                transcript: transcript.clone(),
4243                tools: request_tools,
4244                tool_choice: request_tool_choice,
4245                reasoning: request_reasoning,
4246                output: OutputConfig::default(),
4247                runtime: RuntimeHints {
4248                    auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4249                    profile: runtime_profile,
4250                    parallel_tool_calls: Some(parallel_tool_calls),
4251                    hosted_web_search: cfg.hosted_web_search.clone(),
4252                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4253                    speed_policy: speed_policy_decision,
4254                    reliability: Some(cfg.reliability.clone().into()),
4255                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4256                    ..RuntimeHints::default()
4257                },
4258                metadata: request_metadata,
4259            };
4260
4261            let ctx = InferenceTurnContext {
4262                thread_id: &req.thread_id,
4263                turn_id: &turn_id,
4264                tool_executor: Some(std::sync::Arc::new(
4265                    crate::tool_execution::RuntimeTurnToolExecutor {
4266                        runtime: Arc::clone(self),
4267                        thread_id: req.thread_id.clone(),
4268                        turn_id: turn_id.clone(),
4269                        workspace: Some(workspace.clone()),
4270                        deadline: turn_deadline,
4271                    },
4272                )),
4273            };
4274            let stream_future = engine.stream_turn(ctx, request);
4275            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4276                turn_deadline,
4277                runtime_profile,
4278                req.task_ledger_required,
4279                deadline_finalization_reserve,
4280                deadline_finalization_requested || deadline_scoreable_completion_requested,
4281                task_ledger_scoreable_checkpoints,
4282                &transcript,
4283            ) {
4284                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4285                    Ok(stream) => match stream {
4286                        Ok(stream) => stream,
4287                        Err(err) => {
4288                            let error = err.to_string();
4289                            if self
4290                                .try_recover_from_context_limit(
4291                                    &req.thread_id,
4292                                    &turn_id,
4293                                    &provider,
4294                                    &model,
4295                                    &error,
4296                                    &mut transcript,
4297                                    &mut compacted_this_turn,
4298                                    &mut context_limit_recovery_attempts,
4299                                )
4300                                .await?
4301                            {
4302                                continue 'tool_rounds;
4303                            }
4304                            return Err(err);
4305                        }
4306                    },
4307                    Err(_) => {
4308                        if runtime_profile == RuntimeProfile::Eval
4309                            && !deadline_finalization_requested
4310                        {
4311                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4312                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4313                                && task_ledger_scoreable_checkpoints
4314                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4315                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4316                            {
4317                                task_ledger_scoreable_checkpoints += 1;
4318                                let item = TranscriptItem::UserMessage(UserMessage::text(
4319                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4320                                ));
4321                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4322                                    .await?;
4323                                transcript.push(item);
4324                                continue 'tool_rounds;
4325                            }
4326                            self.start_deadline_finalization(
4327                                &req.thread_id,
4328                                &turn_id,
4329                                &mut transcript,
4330                                remaining,
4331                            )
4332                            .await?;
4333                            deadline_finalization_requested = true;
4334                            continue 'tool_rounds;
4335                        }
4336                        self.fail_turn_due_to_deadline(
4337                            &req.thread_id,
4338                            &turn_id,
4339                            deadline,
4340                            &transcript,
4341                        )
4342                        .await?;
4343                        return Ok(TurnRunOutcome::Stopped);
4344                    }
4345                }
4346            } else {
4347                match stream_future.await {
4348                    Ok(stream) => stream,
4349                    Err(err) => {
4350                        let error = err.to_string();
4351                        if self
4352                            .try_recover_from_context_limit(
4353                                &req.thread_id,
4354                                &turn_id,
4355                                &provider,
4356                                &model,
4357                                &error,
4358                                &mut transcript,
4359                                &mut compacted_this_turn,
4360                                &mut context_limit_recovery_attempts,
4361                            )
4362                            .await?
4363                        {
4364                            continue 'tool_rounds;
4365                        }
4366                        return Err(err);
4367                    }
4368                }
4369            };
4370            let mut assistant_text = String::new();
4371            let mut phase_messages = Vec::<AssistantMessage>::new();
4372            let mut reasoning_text = String::new();
4373            let mut tool_calls = Vec::new();
4374            let mut provider_metadata = None;
4375            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4376            // sometimes abort the SSE stream after emitting the compaction item
4377            // ("error decoding response body") before response.completed, so we
4378            // must not rely solely on ProviderMetadata for the boundary.
4379            let mut stream_compaction_item: Option<serde_json::Value> = None;
4380
4381            loop {
4382                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4383                    turn_deadline,
4384                    runtime_profile,
4385                    req.task_ledger_required,
4386                    deadline_finalization_reserve,
4387                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4388                    task_ledger_scoreable_checkpoints,
4389                    &transcript,
4390                ) {
4391                    match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4392                        Ok(next) => next,
4393                        Err(_) => {
4394                            if runtime_profile == RuntimeProfile::Eval
4395                                && !deadline_finalization_requested
4396                            {
4397                                let remaining =
4398                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4399                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4400                                    && task_ledger_scoreable_checkpoints
4401                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4402                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4403                                {
4404                                    task_ledger_scoreable_checkpoints += 1;
4405                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4406                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4407                                    ));
4408                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4409                                        .await?;
4410                                    transcript.push(item);
4411                                    continue 'tool_rounds;
4412                                }
4413                                self.start_deadline_finalization(
4414                                    &req.thread_id,
4415                                    &turn_id,
4416                                    &mut transcript,
4417                                    remaining,
4418                                )
4419                                .await?;
4420                                deadline_finalization_requested = true;
4421                                continue 'tool_rounds;
4422                            }
4423                            self.fail_turn_due_to_deadline(
4424                                &req.thread_id,
4425                                &turn_id,
4426                                deadline,
4427                                &transcript,
4428                            )
4429                            .await?;
4430                            return Ok(TurnRunOutcome::Stopped);
4431                        }
4432                    }
4433                } else {
4434                    stream.next().await
4435                };
4436                let Some(res) = next else {
4437                    break;
4438                };
4439                let event = match res {
4440                    Ok(event) => event,
4441                    Err(err) => {
4442                        let error = err.to_string();
4443                        if runtime_profile == RuntimeProfile::Eval
4444                            && !deadline_finalization_requested
4445                            && let Some(cause) = provider_stream_retry_cause(&error)
4446                        {
4447                            let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4448                            let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4449                            if retry_attempt < policy.provider_retry_max_attempts {
4450                                provider_stream_retry_attempts = retry_attempt;
4451                                let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4452                                self.emit(RoderEvent::ReliabilityRetryRecorded(
4453                                    ReliabilityRetryRecorded {
4454                                        context: ReliabilityContext {
4455                                            thread_id: req.thread_id.clone(),
4456                                            turn_id: turn_id.clone(),
4457                                            provider: Some(provider.clone()),
4458                                            model: Some(model.clone()),
4459                                            ..ReliabilityContext::default()
4460                                        },
4461                                        error_class: ReliabilityErrorClass::ProviderError,
4462                                        decision: ReliabilityRetryDecision::Retry,
4463                                        attempt: retry_attempt,
4464                                        max_attempts: policy.provider_retry_max_attempts,
4465                                        delay_ms: Some(delay_ms),
4466                                        details: ReliabilityDetails::redacted(format!(
4467                                            "{cause}: {error}"
4468                                        )),
4469                                        timestamp: OffsetDateTime::now_utc(),
4470                                    },
4471                                ))
4472                                .await;
4473                                if delay_ms > 0 {
4474                                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4475                                        .await;
4476                                }
4477                                continue 'tool_rounds;
4478                            }
4479                        }
4480                        if self
4481                            .try_recover_from_context_limit(
4482                                &req.thread_id,
4483                                &turn_id,
4484                                &provider,
4485                                &model,
4486                                &error,
4487                                &mut transcript,
4488                                &mut compacted_this_turn,
4489                                &mut context_limit_recovery_attempts,
4490                            )
4491                            .await?
4492                        {
4493                            continue 'tool_rounds;
4494                        }
4495                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4496                            thread_id: req.thread_id.clone(),
4497                            turn_id: turn_id.clone(),
4498                            error: error.clone(),
4499                            error_kind: None,
4500                            usage: None,
4501                            timestamp: OffsetDateTime::now_utc(),
4502                        }))
4503                        .await;
4504                        self.complete_team_member_turn_with_result(
4505                            &req.thread_id,
4506                            &turn_id,
4507                            TeamMemberStatus::Failed,
4508                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4509                            Some(error),
4510                        )
4511                        .await?;
4512                        return Err(err);
4513                    }
4514                };
4515
4516                let inference_timestamp = OffsetDateTime::now_utc();
4517                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4518                    thread_id: req.thread_id.clone(),
4519                    turn_id: turn_id.clone(),
4520                    event: event.clone(),
4521                    timestamp: inference_timestamp,
4522                }))
4523                .await;
4524
4525                match event {
4526                    InferenceEvent::MessageDelta(delta) => {
4527                        if let Some((team_id, member)) =
4528                            self.teams.member_for_thread(&req.thread_id).await
4529                        {
4530                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4531                                team_id,
4532                                member_id: member.id,
4533                                member_thread_id: req.thread_id.clone(),
4534                                turn_id: turn_id.clone(),
4535                                delta: delta.text.clone(),
4536                                timestamp: OffsetDateTime::now_utc(),
4537                            }))
4538                            .await;
4539                        }
4540                        if is_final_answer_phase(delta.phase.as_deref()) {
4541                            assistant_text.push_str(&delta.text);
4542                        } else if let Some(last) = phase_messages.last_mut()
4543                            && last.phase == delta.phase
4544                        {
4545                            last.text.push_str(&delta.text);
4546                        } else {
4547                            phase_messages.push(AssistantMessage {
4548                                text: delta.text,
4549                                phase: delta.phase,
4550                            });
4551                        }
4552                    }
4553                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4554                    InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4555                    InferenceEvent::Failed(failure) => {
4556                        speed_policy.record_failure();
4557                        let error = failure.message;
4558                        if self
4559                            .try_recover_from_context_limit(
4560                                &req.thread_id,
4561                                &turn_id,
4562                                &provider,
4563                                &model,
4564                                &error,
4565                                &mut transcript,
4566                                &mut compacted_this_turn,
4567                                &mut context_limit_recovery_attempts,
4568                            )
4569                            .await?
4570                        {
4571                            continue 'tool_rounds;
4572                        }
4573                        self.persist_turn_item(
4574                            &req.thread_id,
4575                            &turn_id,
4576                            &TranscriptItem::Error(ErrorRecord {
4577                                message: error.clone(),
4578                            }),
4579                        )
4580                        .await?;
4581                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4582                            thread_id: req.thread_id.clone(),
4583                            turn_id: turn_id.clone(),
4584                            error: error.clone(),
4585                            error_kind: None,
4586                            usage: None,
4587                            timestamp: OffsetDateTime::now_utc(),
4588                        }))
4589                        .await;
4590                        self.complete_team_member_turn_with_result(
4591                            &req.thread_id,
4592                            &turn_id,
4593                            TeamMemberStatus::Failed,
4594                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4595                            Some(error),
4596                        )
4597                        .await?;
4598                        return Ok(TurnRunOutcome::Stopped);
4599                    }
4600                    InferenceEvent::Usage(usage) => {
4601                        turn_usage.add_assign(&usage);
4602                    }
4603                    InferenceEvent::Completed(metadata) => {
4604                        turn_finish_reason = metadata
4605                            .stop_reason
4606                            .as_deref()
4607                            .map(finish_reason_from_stop_reason);
4608                    }
4609                    InferenceEvent::Compaction(progress) => {
4610                        // Persist the boundary as soon as the provider emits a
4611                        // completed compaction item. ChatGPT Codex often aborts
4612                        // the SSE stream right after ("error decoding response
4613                        // body"), so waiting for ProviderMetadata loses the
4614                        // boundary and every subsequent request re-compacts.
4615                        if progress.status == "completed"
4616                            && let Some(item) = progress.item
4617                        {
4618                            let already = stream_compaction_item
4619                                .as_ref()
4620                                .and_then(|existing| {
4621                                    existing.get("id").and_then(serde_json::Value::as_str)
4622                                })
4623                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4624                                .is_some_and(|(a, b)| a == b);
4625                            if !already {
4626                                stream_compaction_item = Some(item.clone());
4627                                let boundary = TranscriptItem::ProviderMetadata(
4628                                    crate::compaction::provider_metadata_from_compaction_item(item),
4629                                );
4630                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4631                                    .await?;
4632                                transcript.push(boundary);
4633                                transcript =
4634                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4635                                compacted_this_turn = true;
4636                            }
4637                        }
4638                    }
4639                    InferenceEvent::HostedToolCallStarted(_)
4640                    | InferenceEvent::HostedToolCallCompleted(_)
4641                    | InferenceEvent::ToolCallStarted(_)
4642                    | InferenceEvent::ToolCallDelta(_) => {}
4643                    InferenceEvent::ProviderMetadata(mut metadata) => {
4644                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4645                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4646                                &mut metadata,
4647                                compaction_item,
4648                            );
4649                        }
4650                        provider_metadata = Some(metadata);
4651                    }
4652                }
4653            }
4654
4655            speed_policy.record_model_output(
4656                !assistant_text.is_empty() || !phase_messages.is_empty(),
4657                tool_calls.len(),
4658            );
4659            if tool_calls.is_empty() {
4660                let steers = self.drain_turn_steers(&turn_id).await;
4661                if !steers.is_empty() {
4662                    for message in phase_messages {
4663                        let item = TranscriptItem::AssistantMessage(message);
4664                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4665                            .await?;
4666                        transcript.push(item);
4667                        self.persist_model_profile_segment(
4668                            &req.thread_id,
4669                            &turn_id,
4670                            model_profile.as_ref(),
4671                            &provider,
4672                            &model,
4673                            "assistant",
4674                        )
4675                        .await?;
4676                    }
4677                    if !assistant_text.is_empty() {
4678                        let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4679                            text: assistant_text,
4680                            phase: Some(FINAL_ANSWER_PHASE.to_string()),
4681                        });
4682                        self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4683                            .await?;
4684                        transcript.push(assistant);
4685                        self.persist_model_profile_segment(
4686                            &req.thread_id,
4687                            &turn_id,
4688                            model_profile.as_ref(),
4689                            &provider,
4690                            &model,
4691                            "assistant",
4692                        )
4693                        .await?;
4694                    }
4695                    if let Some(metadata) = provider_metadata {
4696                        let had_provider_compaction =
4697                            crate::compaction::provider_metadata_has_compaction(&metadata);
4698                        let item = TranscriptItem::ProviderMetadata(metadata);
4699                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4700                            .await?;
4701                        transcript.push(item);
4702                        if had_provider_compaction {
4703                            // Server-side compaction replaced the prior window;
4704                            // drop pre-boundary items so the next request does
4705                            // not re-send (and re-compact) the full history.
4706                            transcript =
4707                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4708                            compacted_this_turn = true;
4709                        }
4710                    }
4711                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4712                        .await?;
4713                    continue;
4714                }
4715                if !deadline_finalization_requested
4716                    && req.task_ledger_required
4717                    && runtime_profile == RuntimeProfile::Eval
4718                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4719                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4720                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4721                {
4722                    task_ledger_completion_reminders += 1;
4723                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4724                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4725                        .await?;
4726                    transcript.push(item);
4727                    continue;
4728                }
4729                if !deadline_finalization_requested
4730                    && let Some(prompt) = verification_gate.blocking_prompt()
4731                {
4732                    speed_policy.record_verification_required();
4733                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4734                        thread_id: req.thread_id.clone(),
4735                        turn_id: turn_id.clone(),
4736                        reason: verification_gate.reason(),
4737                        changed_files: verification_gate.changed_files(),
4738                        tool_evidence: verification_gate.tool_evidence.clone(),
4739                        tests_run: verification_gate.tests_run.clone(),
4740                        open_gaps: verification_gate.open_gaps.clone(),
4741                        timestamp: OffsetDateTime::now_utc(),
4742                    }))
4743                    .await;
4744                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4745                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4746                        .await?;
4747                    transcript.push(item);
4748                    continue;
4749                }
4750                // Loop persistence nudge: in non-interactive/eval runs, when the
4751                // model returns a final message with no tool calls, gently prompt
4752                // it to keep going (bounded by `empty_tool_call_nudges`) before
4753                // ending the turn. Gated to non-interactive/eval so interactive
4754                // chat completions are never delayed, and only fires when the
4755                // model actually produced a final answer to re-examine.
4756                if !deadline_finalization_requested
4757                    && runtime_profile != RuntimeProfile::Interactive
4758                    && cfg.reliability.empty_tool_call_nudges > 0
4759                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4760                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4761                {
4762                    empty_tool_call_nudges_used += 1;
4763                    let item = TranscriptItem::UserMessage(UserMessage::text(
4764                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4765                    ));
4766                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4767                        .await?;
4768                    transcript.push(item);
4769                    continue;
4770                }
4771                if deadline_finalization_requested
4772                    && assistant_text.trim().is_empty()
4773                    && phase_messages.is_empty()
4774                {
4775                    assistant_text = format!(
4776                        "Deadline finalization completed without model text. {}",
4777                        turn_partial_result(&transcript)
4778                    );
4779                }
4780                final_phase_messages = phase_messages;
4781                final_assistant_text = assistant_text;
4782                final_reasoning_text = reasoning_text;
4783                final_provider_metadata = provider_metadata;
4784                exhausted_tool_rounds = false;
4785                break;
4786            }
4787
4788            for message in phase_messages {
4789                let item = TranscriptItem::AssistantMessage(message);
4790                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4791                    .await?;
4792                transcript.push(item);
4793                self.persist_model_profile_segment(
4794                    &req.thread_id,
4795                    &turn_id,
4796                    model_profile.as_ref(),
4797                    &provider,
4798                    &model,
4799                    "assistant",
4800                )
4801                .await?;
4802            }
4803            if !assistant_text.is_empty() {
4804                transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4805                    text: assistant_text,
4806                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4807                }));
4808                self.persist_model_profile_segment(
4809                    &req.thread_id,
4810                    &turn_id,
4811                    model_profile.as_ref(),
4812                    &provider,
4813                    &model,
4814                    "assistant",
4815                )
4816                .await?;
4817            }
4818            if let Some(metadata) = provider_metadata {
4819                let had_provider_compaction =
4820                    crate::compaction::provider_metadata_has_compaction(&metadata);
4821                let item = TranscriptItem::ProviderMetadata(metadata);
4822                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4823                    .await?;
4824                transcript.push(item);
4825                if had_provider_compaction {
4826                    // Server-side compaction replaced the prior window; drop
4827                    // pre-boundary items so subsequent tool rounds use the
4828                    // compact window as the next input.
4829                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4830                    compacted_this_turn = true;
4831                }
4832            }
4833            // Persist CoT before tool calls so providers that require
4834            // `reasoning_content` on tool-call assistant turns (DeepSeek) can
4835            // replay it on the next request. Without this, multi-step tool
4836            // rollouts drop the thinking block and the API returns 400.
4837            if !reasoning_text.is_empty() {
4838                let item = TranscriptItem::ReasoningSummary(ReasoningSummary {
4839                    text: std::mem::take(&mut reasoning_text),
4840                });
4841                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4842                    .await?;
4843                transcript.push(item);
4844            }
4845            for call in &tool_calls {
4846                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4847                    id: call.id.clone(),
4848                    name: call.name.clone(),
4849                    arguments: call.arguments.clone(),
4850                });
4851                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4852                    .await?;
4853                transcript.push(tool_item);
4854                self.persist_model_profile_segment(
4855                    &req.thread_id,
4856                    &turn_id,
4857                    model_profile.as_ref(),
4858                    &provider,
4859                    &model,
4860                    "tool_call",
4861                )
4862                .await?;
4863            }
4864            if let Some(deadline) = turn_deadline
4865                && deadline_expired(deadline)
4866            {
4867                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4868                    .await?;
4869                return Ok(TurnRunOutcome::Stopped);
4870            }
4871            let results = self
4872                .route_tool_calls(
4873                    &req.thread_id,
4874                    &turn_id,
4875                    tool_calls,
4876                    parallel_tool_calls,
4877                    Some(workspace.as_str()),
4878                    turn_deadline,
4879                )
4880                .await?;
4881            let reliability_limit = reliability.record_tool_results(
4882                &cfg.reliability,
4883                &results,
4884                runtime_profile == RuntimeProfile::Interactive,
4885            );
4886            for result in results {
4887                verification_gate.record_tool_result(&result);
4888                transcript.push(TranscriptItem::ToolResult(result));
4889                self.persist_model_profile_segment(
4890                    &req.thread_id,
4891                    &turn_id,
4892                    model_profile.as_ref(),
4893                    &provider,
4894                    &model,
4895                    "tool_result",
4896                )
4897                .await?;
4898            }
4899            if let Some(limit) = reliability_limit {
4900                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4901                    // Loop persistence: rather than ending the turn, reset the
4902                    // consecutive-failure counter, nudge the model to keep going,
4903                    // and continue the round loop. Bounded by the per-turn tool
4904                    // failure ceiling, `max_model_calls_per_turn`, and
4905                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4906                    self.record_reliability_limit_continuation(
4907                        &req.thread_id,
4908                        &turn_id,
4909                        &provider,
4910                        &model,
4911                        limit,
4912                    )
4913                    .await;
4914                    reliability.reset_consecutive_failures();
4915                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4916                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4917                    ));
4918                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4919                        .await?;
4920                    transcript.push(nudge);
4921                } else {
4922                    self.fail_turn_due_to_reliability_limit(
4923                        &req.thread_id,
4924                        &turn_id,
4925                        &provider,
4926                        &model,
4927                        limit,
4928                        &transcript,
4929                    )
4930                    .await?;
4931                    return Ok(TurnRunOutcome::Stopped);
4932                }
4933            }
4934            transcript = self
4935                .compact_transcript_if_needed(
4936                    &req.thread_id,
4937                    &turn_id,
4938                    &provider,
4939                    &model,
4940                    transcript,
4941                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4942                )
4943                .await?;
4944            compacted_this_turn = compacted_this_turn
4945                || transcript
4946                    .iter()
4947                    .any(crate::compaction::is_compaction_boundary);
4948        }
4949
4950        if exhausted_tool_rounds {
4951            let message =
4952                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4953            self.persist_turn_item(
4954                &req.thread_id,
4955                &turn_id,
4956                &TranscriptItem::Error(ErrorRecord {
4957                    message: message.clone(),
4958                }),
4959            )
4960            .await?;
4961            self.emit(RoderEvent::TurnFailed(TurnFailed {
4962                thread_id: req.thread_id.clone(),
4963                turn_id: turn_id.clone(),
4964                error: message.clone(),
4965                error_kind: None,
4966                usage: None,
4967                timestamp: OffsetDateTime::now_utc(),
4968            }))
4969            .await;
4970            self.complete_team_member_turn_with_result(
4971                &req.thread_id,
4972                &turn_id,
4973                TeamMemberStatus::Failed,
4974                None,
4975                Some(message),
4976            )
4977            .await?;
4978            return Ok(TurnRunOutcome::Stopped);
4979        }
4980
4981        if !final_reasoning_text.is_empty() {
4982            self.persist_turn_item(
4983                &req.thread_id,
4984                &turn_id,
4985                &TranscriptItem::ReasoningSummary(ReasoningSummary {
4986                    text: final_reasoning_text,
4987                }),
4988            )
4989            .await?;
4990        }
4991        for message in final_phase_messages {
4992            self.persist_turn_item(
4993                &req.thread_id,
4994                &turn_id,
4995                &TranscriptItem::AssistantMessage(message),
4996            )
4997            .await?;
4998            self.persist_model_profile_segment(
4999                &req.thread_id,
5000                &turn_id,
5001                model_profile.as_ref(),
5002                &provider,
5003                &model,
5004                "assistant",
5005            )
5006            .await?;
5007        }
5008        if !final_assistant_text.is_empty() {
5009            self.persist_turn_item(
5010                &req.thread_id,
5011                &turn_id,
5012                &TranscriptItem::AssistantMessage(AssistantMessage {
5013                    text: final_assistant_text.clone(),
5014                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
5015                }),
5016            )
5017            .await?;
5018            self.persist_model_profile_segment(
5019                &req.thread_id,
5020                &turn_id,
5021                model_profile.as_ref(),
5022                &provider,
5023                &model,
5024                "assistant",
5025            )
5026            .await?;
5027        }
5028        if let Some(metadata) = final_provider_metadata {
5029            self.persist_turn_item(
5030                &req.thread_id,
5031                &turn_id,
5032                &TranscriptItem::ProviderMetadata(metadata),
5033            )
5034            .await?;
5035        }
5036
5037        let turn_usage_tokens = turn_usage.total_tokens as i64;
5038        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5039        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5040            .await?;
5041        self.goals
5042            .account_turn_usage(
5043                &req.thread_id,
5044                turn_usage_tokens,
5045                OffsetDateTime::now_utc() - turn_started_at,
5046            )
5047            .await?;
5048        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5049        self.record_turn_lifecycle_with_ownership(
5050            req.thread_id.clone(),
5051            turn_id.clone(),
5052            TurnLifecycleState::Completed,
5053            cleanup,
5054            None,
5055            ownership,
5056        )
5057        .await;
5058        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5059            thread_id: req.thread_id.clone(),
5060            turn_id: turn_id.clone(),
5061            usage: completed_usage,
5062            finish_reason: turn_finish_reason,
5063            timestamp: OffsetDateTime::now_utc(),
5064        }))
5065        .await;
5066        self.complete_team_member_turn_with_result(
5067            &req.thread_id,
5068            &turn_id,
5069            TeamMemberStatus::Completed,
5070            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5071            None,
5072        )
5073        .await?;
5074        Ok(TurnRunOutcome::Completed)
5075    }
5076
5077    async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
5078        let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
5079            return Vec::new();
5080        };
5081        let mut steers = active.steers.lock().await;
5082        std::mem::take(&mut *steers)
5083    }
5084
5085    async fn route_tool_calls(
5086        self: &Arc<Self>,
5087        thread_id: &ThreadId,
5088        turn_id: &TurnId,
5089        calls: Vec<ToolCallCompleted>,
5090        parallel: bool,
5091        workspace: Option<&str>,
5092        deadline: Option<OffsetDateTime>,
5093    ) -> anyhow::Result<Vec<ToolResultRecord>> {
5094        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
5095        // `agent_swarm` must be the only tool call in a model response. A mixed
5096        // or multi-swarm batch is denied wholesale with actionable retry text so
5097        // the model re-issues `agent_swarm` by itself, and every tool_call_id
5098        // still gets a response (keeping the chat-completions transcript valid).
5099        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5100            calls.iter().map(|call| call.name.as_str()),
5101        ) {
5102            let message = violation.deny_message();
5103            return Ok(calls
5104                .into_iter()
5105                .map(|call| ToolResultRecord {
5106                    id: call.id,
5107                    name: Some(call.name),
5108                    result: message.clone(),
5109                    display_payload: None,
5110                    is_error: true,
5111                })
5112                .collect());
5113        }
5114        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
5115        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
5116        // progress flows through the existing Subagent* trace events.
5117        let swarm_call = (calls.len() == 1
5118            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5119            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5120        if let Some((tool_id, args)) = &swarm_call {
5121            self.emit(RoderEvent::AgentSwarmStarted(
5122                roder_api::subagents::AgentSwarmStarted {
5123                    thread_id: thread_id.clone(),
5124                    turn_id: turn_id.clone(),
5125                    tool_id: tool_id.clone(),
5126                    child_count: agent_swarm_child_count(args),
5127                    timestamp: OffsetDateTime::now_utc(),
5128                },
5129            ))
5130            .await;
5131        }
5132
5133        let force_sequential = calls
5134            .iter()
5135            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5136        let results =
5137            if parallel && !force_sequential {
5138                try_join_all(calls.into_iter().map(|call| {
5139                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5140                }))
5141                .await
5142            } else {
5143                let mut results = Vec::with_capacity(calls.len());
5144                for call in calls {
5145                    results.push(
5146                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5147                            .await?,
5148                    );
5149                }
5150                Ok(results)
5151            }?;
5152
5153        if let Some((tool_id, _)) = &swarm_call
5154            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5155            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5156        {
5157            self.emit(RoderEvent::AgentSwarmCompleted(
5158                roder_api::subagents::AgentSwarmCompleted {
5159                    thread_id: thread_id.clone(),
5160                    turn_id: turn_id.clone(),
5161                    tool_id: tool_id.clone(),
5162                    completed,
5163                    failed,
5164                    aborted,
5165                    timestamp: OffsetDateTime::now_utc(),
5166                },
5167            ))
5168            .await;
5169        }
5170
5171        Ok(results)
5172    }
5173
5174    /// On provider context/prompt-length failures, force-compact (and if needed
5175    /// strip the last bulky item) then retry the current tool round instead of
5176    /// failing the turn. Bounded so a permanently oversized suffix still stops.
5177    async fn try_recover_from_context_limit(
5178        &self,
5179        thread_id: &ThreadId,
5180        turn_id: &TurnId,
5181        provider: &str,
5182        model: &str,
5183        error: &str,
5184        transcript: &mut Vec<TranscriptItem>,
5185        compacted_this_turn: &mut bool,
5186        recovery_attempts: &mut u32,
5187    ) -> anyhow::Result<bool> {
5188        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5189        if !crate::compaction::is_context_limit_failure_message(error) {
5190            return Ok(false);
5191        }
5192        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5193            return Ok(false);
5194        }
5195        *recovery_attempts = recovery_attempts.saturating_add(1);
5196
5197        let error_item = TranscriptItem::Error(ErrorRecord {
5198            message: error.to_string(),
5199        });
5200        self.persist_turn_item(thread_id, turn_id, &error_item)
5201            .await?;
5202        transcript.push(error_item);
5203
5204        // After the first failed recovery, drop the item ahead of the latest
5205        // user/error so a second compact can succeed when the suffix itself is
5206        // still oversized (e.g. a huge tool result right before the prompt).
5207        if *recovery_attempts > 1 {
5208            *transcript =
5209                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5210        }
5211
5212        let force_options = crate::compaction::CompactionOptions {
5213            allow_repeat: true,
5214            force: true,
5215            hysteresis_baseline: None,
5216            preserve_hint: Some(
5217                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5218                    .to_string(),
5219            ),
5220        };
5221        let before_len = transcript.len();
5222        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5223        *transcript = self
5224            .compact_transcript_if_needed(
5225                thread_id,
5226                turn_id,
5227                provider,
5228                model,
5229                std::mem::take(transcript),
5230                force_options,
5231            )
5232            .await?;
5233        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5234        *compacted_this_turn = *compacted_this_turn
5235            || transcript
5236                .iter()
5237                .any(crate::compaction::is_compaction_boundary);
5238
5239        self.emit(RoderEvent::ReliabilityRetryRecorded(
5240            ReliabilityRetryRecorded {
5241                context: ReliabilityContext {
5242                    thread_id: thread_id.clone(),
5243                    turn_id: turn_id.clone(),
5244                    provider: Some(provider.to_string()),
5245                    model: Some(model.to_string()),
5246                    ..ReliabilityContext::default()
5247                },
5248                error_class: ReliabilityErrorClass::ProviderError,
5249                decision: ReliabilityRetryDecision::Retry,
5250                attempt: *recovery_attempts,
5251                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5252                delay_ms: Some(0),
5253                details: ReliabilityDetails::redacted(format!(
5254                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5255                    transcript.len()
5256                )),
5257                timestamp: OffsetDateTime::now_utc(),
5258            },
5259        ))
5260        .await;
5261
5262        // If force-compact did not shrink anything and we still have budget,
5263        // strip once more so the next round is not identical.
5264        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5265        {
5266            *transcript =
5267                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5268        }
5269        Ok(true)
5270    }
5271
5272    async fn fail_turn_with_error(
5273        &self,
5274        thread_id: &ThreadId,
5275        turn_id: &TurnId,
5276        message: String,
5277    ) -> anyhow::Result<()> {
5278        self.persist_turn_item(
5279            thread_id,
5280            turn_id,
5281            &TranscriptItem::Error(ErrorRecord {
5282                message: message.clone(),
5283            }),
5284        )
5285        .await?;
5286        self.emit(RoderEvent::TurnFailed(TurnFailed {
5287            thread_id: thread_id.clone(),
5288            turn_id: turn_id.clone(),
5289            error: message.clone(),
5290            error_kind: None,
5291            usage: None,
5292            timestamp: OffsetDateTime::now_utc(),
5293        }))
5294        .await;
5295        self.complete_team_member_turn_with_result(
5296            thread_id,
5297            turn_id,
5298            TeamMemberStatus::Failed,
5299            None,
5300            Some(message),
5301        )
5302        .await?;
5303        Ok(())
5304    }
5305
5306    async fn fail_turn_due_to_deadline(
5307        &self,
5308        thread_id: &ThreadId,
5309        turn_id: &TurnId,
5310        deadline: OffsetDateTime,
5311        transcript: &[TranscriptItem],
5312    ) -> anyhow::Result<()> {
5313        let partial_result = turn_partial_result(transcript);
5314        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5315            thread_id: thread_id.clone(),
5316            turn_id: turn_id.clone(),
5317            summary: partial_result.clone(),
5318            timestamp: OffsetDateTime::now_utc(),
5319        }))
5320        .await;
5321        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5322            thread_id: thread_id.clone(),
5323            turn_id: turn_id.clone(),
5324            deadline,
5325            partial_result: partial_result.clone(),
5326            timestamp: OffsetDateTime::now_utc(),
5327        }))
5328        .await;
5329        let message = "turn deadline expired".to_string();
5330        self.persist_turn_item(
5331            thread_id,
5332            turn_id,
5333            &TranscriptItem::Error(ErrorRecord {
5334                message: format!("{message}: {partial_result}"),
5335            }),
5336        )
5337        .await?;
5338        self.emit(RoderEvent::TurnFailed(TurnFailed {
5339            thread_id: thread_id.clone(),
5340            turn_id: turn_id.clone(),
5341            error: message.clone(),
5342            error_kind: Some("deadline_timeout".to_string()),
5343            usage: None,
5344            timestamp: OffsetDateTime::now_utc(),
5345        }))
5346        .await;
5347        self.complete_team_member_turn_with_result(
5348            thread_id,
5349            turn_id,
5350            TeamMemberStatus::Failed,
5351            None,
5352            Some(format!("{message}: {partial_result}")),
5353        )
5354        .await?;
5355        Ok(())
5356    }
5357
5358    async fn start_deadline_finalization(
5359        &self,
5360        thread_id: &ThreadId,
5361        turn_id: &TurnId,
5362        transcript: &mut Vec<TranscriptItem>,
5363        remaining_seconds: u64,
5364    ) -> anyhow::Result<()> {
5365        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5366            remaining_seconds,
5367        ));
5368        self.persist_turn_item(thread_id, turn_id, &item).await?;
5369        transcript.push(item);
5370        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5371            thread_id: thread_id.clone(),
5372            turn_id: turn_id.clone(),
5373            summary: turn_partial_result(transcript),
5374            timestamp: OffsetDateTime::now_utc(),
5375        }))
5376        .await;
5377        Ok(())
5378    }
5379
5380    /// Emits the reliability-limit event for a continuation (loop persistence)
5381    /// without failing the turn. The caller resets the failure counter and
5382    /// injects a nudge so the round loop keeps going.
5383    async fn record_reliability_limit_continuation(
5384        &self,
5385        thread_id: &ThreadId,
5386        turn_id: &TurnId,
5387        provider: &str,
5388        model: &str,
5389        limit: ReliabilityLimitHit,
5390    ) {
5391        self.emit(RoderEvent::ReliabilityLimitRecorded(
5392            ReliabilityLimitRecorded {
5393                context: ReliabilityContext {
5394                    thread_id: thread_id.clone(),
5395                    turn_id: turn_id.clone(),
5396                    tool_id: None,
5397                    tool_name: None,
5398                    provider: Some(provider.to_string()),
5399                    model: Some(model.to_string()),
5400                },
5401                error_class: limit.error_class,
5402                limit_kind: limit.limit_kind,
5403                decision: limit.decision,
5404                current: limit.current,
5405                limit: limit.limit,
5406                details: ReliabilityDetails::redacted(&limit.message),
5407                timestamp: OffsetDateTime::now_utc(),
5408            },
5409        ))
5410        .await;
5411    }
5412
5413    async fn fail_turn_due_to_reliability_limit(
5414        &self,
5415        thread_id: &ThreadId,
5416        turn_id: &TurnId,
5417        provider: &str,
5418        model: &str,
5419        limit: ReliabilityLimitHit,
5420        transcript: &[TranscriptItem],
5421    ) -> anyhow::Result<()> {
5422        self.emit(RoderEvent::ReliabilityLimitRecorded(
5423            ReliabilityLimitRecorded {
5424                context: ReliabilityContext {
5425                    thread_id: thread_id.clone(),
5426                    turn_id: turn_id.clone(),
5427                    tool_id: None,
5428                    tool_name: None,
5429                    provider: Some(provider.to_string()),
5430                    model: Some(model.to_string()),
5431                },
5432                error_class: limit.error_class,
5433                limit_kind: limit.limit_kind,
5434                decision: limit.decision,
5435                current: limit.current,
5436                limit: limit.limit,
5437                details: ReliabilityDetails::redacted(&limit.message),
5438                timestamp: OffsetDateTime::now_utc(),
5439            },
5440        ))
5441        .await;
5442        let partial_result = turn_partial_result(transcript);
5443        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5444            thread_id: thread_id.clone(),
5445            turn_id: turn_id.clone(),
5446            summary: partial_result.clone(),
5447            timestamp: OffsetDateTime::now_utc(),
5448        }))
5449        .await;
5450        let message = format!("reliability limit reached: {}", limit.message);
5451        self.persist_turn_item(
5452            thread_id,
5453            turn_id,
5454            &TranscriptItem::Error(ErrorRecord {
5455                message: format!("{message}: {partial_result}"),
5456            }),
5457        )
5458        .await?;
5459        self.emit(RoderEvent::TurnFailed(TurnFailed {
5460            thread_id: thread_id.clone(),
5461            turn_id: turn_id.clone(),
5462            error: message.clone(),
5463            error_kind: Some("reliability_limit".to_string()),
5464            usage: None,
5465            timestamp: OffsetDateTime::now_utc(),
5466        }))
5467        .await;
5468        self.complete_team_member_turn_with_result(
5469            thread_id,
5470            turn_id,
5471            TeamMemberStatus::Failed,
5472            None,
5473            Some(format!("{message}: {partial_result}")),
5474        )
5475        .await?;
5476        Ok(())
5477    }
5478
5479    async fn append_steers(
5480        &self,
5481        req: &StartTurnRequest,
5482        turn_id: &TurnId,
5483        transcript: &mut Vec<TranscriptItem>,
5484        steers: Vec<QueuedTurnSteer>,
5485    ) -> anyhow::Result<()> {
5486        for queued in steers {
5487            let mut steer = queued.message;
5488            steer.text = steer.text.trim().to_string();
5489            if steer.text.is_empty() && steer.images.is_empty() {
5490                continue;
5491            }
5492            let item = TranscriptItem::UserMessage(steer);
5493            self.persist_turn_item(&req.thread_id, turn_id, &item)
5494                .await?;
5495            transcript.push(item);
5496            if let Some(ack) = queued.mailbox_ack {
5497                self.teams
5498                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5499                    .await?;
5500            }
5501        }
5502        Ok(())
5503    }
5504
5505    async fn persist_model_profile_segment(
5506        &self,
5507        thread_id: &ThreadId,
5508        turn_id: &TurnId,
5509        profile: Option<&ModelHarnessProfile>,
5510        provider: &str,
5511        model: &str,
5512        segment: &str,
5513    ) -> anyhow::Result<()> {
5514        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5515            profile, provider, model, segment,
5516        ));
5517        self.persist_turn_item(thread_id, turn_id, &item).await
5518    }
5519
5520    /**
5521     * Allowlists apply to built-in tools only; external tools are advertised with their
5522     * host-supplied schemas as given. An external tool shadows a built-in with the same name in
5523     * both advertisement and dispatch (see `route_tool_call`).
5524     */
5525    fn filtered_tool_specs(
5526        &self,
5527        cfg: &RuntimeConfig,
5528        model: &str,
5529        profile: Option<&ModelHarnessProfile>,
5530        thread_allowlist: &[String],
5531        external_tools: &[roder_api::tools::ToolSpec],
5532    ) -> Vec<roder_api::tools::ToolSpec> {
5533        let mut specs = self
5534            .tool_registry
5535            .specs_for_edit_tool_with_schema_policy(
5536                edit_tool_for_model(cfg, model),
5537                schema_policy_for_model(profile),
5538            )
5539            .into_iter()
5540            .filter(|spec| {
5541                allowlist_permits(&cfg.tool_allowlist, &spec.name)
5542                    && allowlist_permits(thread_allowlist, &spec.name)
5543                    && !external_tools.iter().any(|tool| tool.name == spec.name)
5544            })
5545            .collect::<Vec<_>>();
5546        specs.extend(external_tools.iter().cloned());
5547        specs
5548    }
5549
5550    fn task_ledger_tool_specs(
5551        &self,
5552        profile: Option<&ModelHarnessProfile>,
5553    ) -> Vec<roder_api::tools::ToolSpec> {
5554        self.tool_registry
5555            .get(TASK_LEDGER_TOOL_NAME)
5556            .map(|tool| {
5557                tool.spec()
5558                    .normalized_for_model_profile(schema_policy_for_model(profile))
5559            })
5560            .into_iter()
5561            .collect()
5562    }
5563
5564    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5565        self.registry
5566            .inference_engine(provider)
5567            .or_else(|| {
5568                self.registry
5569                    .default_inference_engine()
5570                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5571            })
5572            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5573    }
5574
5575    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5576        self.dispatch_local_hook_lifecycle(&event).await;
5577        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5578            let _ = self.persist_turn_lifecycle_record(&record).await;
5579        }
5580        let envelope = self.bus.emit(event);
5581        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5582            && should_persist_thread_event(thread_id)
5583        {
5584            let _ = store.append_event(thread_id, &envelope).await;
5585        }
5586        // Registered event sinks (e.g. process extensions) receive the
5587        // persisted envelope through bounded per-sink queues; a slow sink
5588        // never blocks emit or turn progress.
5589        let dispatcher = self
5590            .event_sink_dispatcher
5591            .get_or_init(|| async {
5592                crate::event_sink_dispatch::EventSinkDispatcher::start(
5593                    &self.registry.event_sinks,
5594                    self.bus.clone(),
5595                )
5596            })
5597            .await;
5598        if !dispatcher.is_empty() {
5599            dispatcher.dispatch(&envelope, &self.bus);
5600        }
5601        envelope
5602    }
5603
5604    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5605        // SessionStart belongs to the session, not the turn: firing it from
5606        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5607        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5608        // way the SessionEnd path does.
5609        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5610        let (thread_id, turn_id, name, matcher, input) = match event {
5611            RoderEvent::ThreadCreated(event) => (
5612                &event.thread_id,
5613                &session_turn_id,
5614                "SessionStart",
5615                Some("startup"),
5616                serde_json::json!({"source":"startup"}),
5617            ),
5618            RoderEvent::ThreadLoaded(event) => (
5619                &event.thread_id,
5620                &session_turn_id,
5621                "SessionStart",
5622                Some("resume"),
5623                serde_json::json!({"source":"resume"}),
5624            ),
5625            RoderEvent::TurnCompleted(event) => (
5626                &event.thread_id,
5627                &event.turn_id,
5628                "Stop",
5629                None,
5630                serde_json::json!({"finishReason":event.finish_reason}),
5631            ),
5632            RoderEvent::TurnFailed(event) => (
5633                &event.thread_id,
5634                &event.turn_id,
5635                "Stop",
5636                None,
5637                serde_json::json!({"error":event.error}),
5638            ),
5639            RoderEvent::ContextCompactionStarted(event) => (
5640                &event.thread_id,
5641                &event.turn_id,
5642                "PreCompact",
5643                None,
5644                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5645            ),
5646            RoderEvent::ContextCompactionRecorded(event) => (
5647                &event.thread_id,
5648                &event.turn_id,
5649                "PostCompact",
5650                None,
5651                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5652            ),
5653            RoderEvent::SubagentStarted(event) => (
5654                &event.parent_thread_id,
5655                &event.parent_turn_id,
5656                "SubagentStart",
5657                Some(event.agent_type.as_str()),
5658                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5659            ),
5660            RoderEvent::SubagentCompleted(event) => (
5661                &event.parent_thread_id,
5662                &event.parent_turn_id,
5663                "SubagentStop",
5664                Some(event.agent_type.as_str()),
5665                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5666            ),
5667            RoderEvent::SubagentFailed(event) => (
5668                &event.parent_thread_id,
5669                &event.parent_turn_id,
5670                "SubagentStop",
5671                Some(event.agent_type.as_str()),
5672                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5673            ),
5674            _ => return,
5675        };
5676        // One SessionStart per session, whichever event opened it.
5677        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5678            return;
5679        }
5680        let workspace = self.config.read().await.workspace.clone();
5681        crate::hooks::run_lifecycle(
5682            self,
5683            thread_id,
5684            turn_id,
5685            workspace.as_deref(),
5686            name,
5687            matcher,
5688            input,
5689        )
5690        .await;
5691    }
5692
5693    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5694    /// this caller won it.
5695    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5696        self.session_hook_started
5697            .write()
5698            .await
5699            .insert(thread_id.clone())
5700    }
5701
5702    /// Records a projected thread item event and persists it through the configured thread store.
5703    ///
5704    /// App-server protocol notification bridges currently call this after translating runtime
5705    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5706    /// must make the same call if they need the derived item event stream persisted.
5707    pub async fn record_thread_item_event_kind(
5708        &self,
5709        thread_id: &ThreadId,
5710        turn_id: &TurnId,
5711        timestamp: OffsetDateTime,
5712        kind: ThreadItemEventKind,
5713    ) -> anyhow::Result<ThreadItemEvent> {
5714        let seq = self.next_thread_item_event_seq(thread_id).await?;
5715        let item_event = ThreadItemEvent {
5716            seq,
5717            event_id: format!("{turn_id}-item-event-{seq}"),
5718            thread_id: thread_id.clone(),
5719            turn_id: turn_id.clone(),
5720            timestamp,
5721            event: kind,
5722        };
5723        if let Some(store) = &self.thread_store {
5724            store.append_item_event(thread_id, &item_event).await?;
5725        }
5726        self.remember_thread_item_event(&item_event).await?;
5727        Ok(item_event)
5728    }
5729
5730    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5731        self.ensure_thread_item_cache(thread_id).await?;
5732        Ok(self
5733            .thread_item_cache
5734            .lock()
5735            .await
5736            .next_item_event_seq(thread_id))
5737    }
5738
5739    pub async fn thread_item_exists(
5740        &self,
5741        thread_id: &ThreadId,
5742        turn_id: &TurnId,
5743        item_id: &str,
5744    ) -> anyhow::Result<bool> {
5745        self.ensure_thread_item_cache(thread_id).await?;
5746        Ok(self
5747            .thread_item_cache
5748            .lock()
5749            .await
5750            .thread_item_exists(thread_id, turn_id, item_id))
5751    }
5752
5753    pub async fn current_reasoning_item_id(
5754        &self,
5755        thread_id: &ThreadId,
5756        turn_id: &TurnId,
5757    ) -> anyhow::Result<Option<String>> {
5758        self.ensure_thread_item_cache(thread_id).await?;
5759        Ok(self
5760            .thread_item_cache
5761            .lock()
5762            .await
5763            .current_reasoning_item_id(thread_id, turn_id))
5764    }
5765
5766    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5767        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5768        self.thread_item_cache
5769            .lock()
5770            .await
5771            .remember_item_event(item_event);
5772        Ok(())
5773    }
5774
5775    pub async fn latest_transcript_item_index(
5776        &self,
5777        thread_id: &ThreadId,
5778        turn_id: &TurnId,
5779    ) -> anyhow::Result<Option<usize>> {
5780        self.ensure_thread_item_cache(thread_id).await?;
5781        Ok(self
5782            .thread_item_cache
5783            .lock()
5784            .await
5785            .latest_transcript_item_index(thread_id, turn_id))
5786    }
5787
5788    async fn next_transcript_item_index(
5789        &self,
5790        thread_id: &ThreadId,
5791        turn_id: &TurnId,
5792    ) -> anyhow::Result<usize> {
5793        self.ensure_thread_item_cache(thread_id).await?;
5794        Ok(self
5795            .thread_item_cache
5796            .lock()
5797            .await
5798            .next_transcript_item_index(thread_id, turn_id))
5799    }
5800
5801    async fn remember_transcript_item_index(
5802        &self,
5803        thread_id: &ThreadId,
5804        turn_id: &TurnId,
5805        item_index: usize,
5806    ) -> anyhow::Result<()> {
5807        self.ensure_thread_item_cache(thread_id).await?;
5808        self.thread_item_cache
5809            .lock()
5810            .await
5811            .remember_transcript_item_index(thread_id, turn_id, item_index);
5812        Ok(())
5813    }
5814
5815    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5816        if self
5817            .thread_item_cache
5818            .lock()
5819            .await
5820            .contains_thread(thread_id)
5821        {
5822            return Ok(());
5823        }
5824
5825        let snapshot = if let Some(store) = &self.thread_store {
5826            store.load_thread(thread_id).await?
5827        } else {
5828            None
5829        };
5830        self.thread_item_cache.lock().await.ensure_thread(
5831            thread_id,
5832            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5833        );
5834        Ok(())
5835    }
5836
5837    pub(crate) async fn persist_turn_item(
5838        &self,
5839        thread_id: &ThreadId,
5840        turn_id: &TurnId,
5841        item: &TranscriptItem,
5842    ) -> anyhow::Result<()> {
5843        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5844        let timestamp = OffsetDateTime::now_utc();
5845        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5846            thread_id: thread_id.clone(),
5847            turn_id: turn_id.clone(),
5848            item_type: match item {
5849                TranscriptItem::UserMessage(_) => "user_message",
5850                TranscriptItem::AssistantMessage(_) => "assistant_message",
5851                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5852                TranscriptItem::ToolCall(_) => "tool_call",
5853                TranscriptItem::ToolResult(_) => "tool_result",
5854                TranscriptItem::FileChange(_) => "file_change",
5855                TranscriptItem::ContextCompaction(_) => "context_compaction",
5856                TranscriptItem::Error(_) => "error",
5857                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5858            }
5859            .to_string(),
5860            item_index: Some(item_index),
5861            item: Some(item.clone()),
5862            timestamp,
5863        }))
5864        .await;
5865        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5866            .await?;
5867        Ok(())
5868    }
5869}
5870
5871fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5872    transcript.iter().any(|item| {
5873        matches!(
5874            item,
5875            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5876        )
5877    })
5878}
5879
5880fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5881    transcript.iter().any(|item| {
5882        matches!(
5883            item,
5884            TranscriptItem::ToolResult(result)
5885                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5886        )
5887    })
5888}
5889
5890fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5891    let latest = transcript.iter().rev().find_map(|item| match item {
5892        TranscriptItem::ToolResult(result)
5893            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5894        {
5895            Some(result.result.as_str())
5896        }
5897        _ => None,
5898    })?;
5899    if !task_ledger_has_open_items(latest) {
5900        return None;
5901    }
5902
5903    let mut ledger = latest.chars().take(1500).collect::<String>();
5904    if latest.chars().nth(1500).is_some() {
5905        ledger.push_str("...");
5906    }
5907    Some(format!(
5908        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5909    ))
5910}
5911
5912fn task_ledger_deadline_completion_prompt(
5913    remaining_seconds: u64,
5914    reserve_seconds: u64,
5915    completion_prompt: &str,
5916) -> String {
5917    format!(
5918        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5919    )
5920}
5921
5922fn task_ledger_scoreable_checkpoint_prompt(
5923    remaining_seconds: u64,
5924    completion_prompt: &str,
5925) -> String {
5926    format!(
5927        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5928    )
5929}
5930
5931fn task_ledger_has_open_items(ledger: &str) -> bool {
5932    ledger.lines().any(|line| {
5933        let line = line.trim_start();
5934        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5935    })
5936}
5937
5938fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5939    cfg.turn_deadline_seconds
5940        .filter(|seconds| *seconds > 0)
5941        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5942}
5943
5944fn deadline_expired(deadline: OffsetDateTime) -> bool {
5945    OffsetDateTime::now_utc() >= deadline
5946}
5947
5948pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5949    let deadline = deadline?;
5950    if deadline <= OffsetDateTime::now_utc() {
5951        return Some(0);
5952    }
5953    Some(
5954        (deadline - OffsetDateTime::now_utc())
5955            .unsigned_abs()
5956            .as_secs()
5957            .max(1),
5958    )
5959}
5960
5961fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5962    let now = OffsetDateTime::now_utc();
5963    if deadline <= now {
5964        return tokio::time::Instant::now();
5965    }
5966    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5967}
5968
5969fn inference_timeout_deadline(
5970    deadline: Option<OffsetDateTime>,
5971    runtime_profile: RuntimeProfile,
5972    task_ledger_required: bool,
5973    reserve_seconds: u64,
5974    finalization_requested: bool,
5975    task_ledger_scoreable_checkpoints: u8,
5976    transcript: &[TranscriptItem],
5977) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5978    let deadline = deadline?;
5979    if runtime_profile == RuntimeProfile::Eval
5980        && task_ledger_required
5981        && !finalization_requested
5982        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5983        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5984        && task_ledger_completion_prompt(transcript).is_some()
5985    {
5986        let checkpoint_deadline =
5987            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5988        if checkpoint_deadline > OffsetDateTime::now_utc() {
5989            return Some((
5990                checkpoint_deadline,
5991                InferenceTimeoutAction::ScoreableCheckpoint,
5992            ));
5993        }
5994    }
5995    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5996        return Some((
5997            deadline - Duration::seconds(reserve_seconds as i64),
5998            InferenceTimeoutAction::Finalization,
5999        ));
6000    }
6001    Some((deadline, InferenceTimeoutAction::Finalization))
6002}
6003
6004fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6005    let tool_results = transcript
6006        .iter()
6007        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6008        .count();
6009    let assistant_messages = transcript
6010        .iter()
6011        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6012        .count();
6013    format!(
6014        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6015        transcript.len()
6016    )
6017}
6018
6019fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6020    let level = effective_reasoning_for_model(cfg, model);
6021    match level.as_str() {
6022        "" | REASONING_NONE => ReasoningConfig::default(),
6023        level => ReasoningConfig {
6024            enabled: true,
6025            level: Some(level.to_string()),
6026        },
6027    }
6028}
6029
6030fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6031    let entry = lookup_model(model)?;
6032    if !entry.supports_compaction {
6033        return None;
6034    }
6035    cfg.auto_compact_token_limit
6036        .or_else(|| {
6037            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6038        })
6039        .or(Some(entry.auto_compact_token_limit))
6040        .filter(|threshold| *threshold > 0)
6041}
6042
6043pub(crate) fn tool_search_for_provider_model(
6044    cfg: &RuntimeConfig,
6045    provider: &str,
6046    model: &str,
6047) -> ToolSearchConfig {
6048    let mut resolved = cfg.tool_search.clone();
6049    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6050        provider_config.apply_to(&mut resolved);
6051    }
6052    if let Some(model_config) = cfg.model_tool_search.get(model) {
6053        model_config.apply_to(&mut resolved);
6054    }
6055    resolved
6056}
6057
6058fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6059    cfg.model_parallel_tool_calls
6060        .get(model)
6061        .copied()
6062        .or_else(|| {
6063            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6064        })
6065        .unwrap_or(true)
6066}
6067
6068/// Count the children an `agent_swarm` call will launch from its arguments
6069/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
6070fn agent_swarm_child_count(arguments: &str) -> usize {
6071    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6072        .map(|request| request.items.len() + request.resume_agent_ids.len())
6073        .unwrap_or(0)
6074}
6075
6076/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
6077/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
6078/// buckets default to 0. Returns `None` when the text is not a swarm result.
6079fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6080    if !text.contains("<agent_swarm_result>") {
6081        return None;
6082    }
6083    let bucket = |label: &str| -> usize {
6084        let needle = format!("{label}: ");
6085        text.find(&needle)
6086            .map(|start| start + needle.len())
6087            .map(|start| {
6088                text[start..]
6089                    .chars()
6090                    .take_while(|c| c.is_ascii_digit())
6091                    .collect::<String>()
6092            })
6093            .and_then(|digits| digits.parse().ok())
6094            .unwrap_or(0)
6095    };
6096    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6097}
6098
6099fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6100    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6101    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6102        return ultracode_reasoning_level_for_model(
6103            model,
6104            &cfg.speed_policy.ultracode_reasoning,
6105            &base_reasoning,
6106        );
6107    }
6108    base_reasoning
6109}
6110
6111fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6112    let Some(entry) = lookup_model(model) else {
6113        return cfg
6114            .reasoning
6115            .clone()
6116            .unwrap_or_else(|| REASONING_NONE.to_string());
6117    };
6118    if entry.supported_reasoning.is_empty() {
6119        return REASONING_NONE.to_string();
6120    }
6121    cfg.reasoning
6122        .as_deref()
6123        .filter(|reasoning| {
6124            entry
6125                .supported_reasoning
6126                .iter()
6127                .any(|option| option.effort == *reasoning)
6128        })
6129        .map(str::to_string)
6130        .or_else(|| {
6131            model_profile_for_model(cfg, model)
6132                .and_then(|profile| profile.reasoning.orientation)
6133                .filter(|reasoning| {
6134                    entry
6135                        .supported_reasoning
6136                        .iter()
6137                        .any(|option| option.effort == reasoning)
6138                })
6139        })
6140        .unwrap_or_else(|| entry.default_reasoning.to_string())
6141}
6142
6143fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6144    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6145        return Ok(());
6146    }
6147    let Some(entry) = lookup_model(model) else {
6148        return Ok(());
6149    };
6150    if entry
6151        .supported_reasoning
6152        .iter()
6153        .any(|option| option.effort == effort)
6154    {
6155        Ok(())
6156    } else {
6157        anyhow::bail!("model {model} does not support reasoning effort {effort}")
6158    }
6159}
6160
6161fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6162    let Some(reasoning) = cfg.reasoning.as_deref() else {
6163        return Ok(());
6164    };
6165    let Some(entry) = lookup_model(&cfg.default_model) else {
6166        return Ok(());
6167    };
6168    if entry.provider != PROVIDER_GEMINI {
6169        return Ok(());
6170    }
6171    validate_reasoning_effort(&cfg.default_model, reasoning)
6172}
6173
6174fn validate_runtime_inference_router_config(
6175    registry: &ExtensionRegistry,
6176    cfg: &RuntimeConfig,
6177) -> anyhow::Result<()> {
6178    if !cfg.inference_router.enabled {
6179        return Ok(());
6180    }
6181    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6182        anyhow::bail!("inference_router.enabled requires inference_router.router");
6183    };
6184    if registry.inference_router(router_id).is_some() {
6185        return Ok(());
6186    }
6187    let available = registry
6188        .inference_routers
6189        .iter()
6190        .map(|router| router.id())
6191        .collect::<Vec<_>>()
6192        .join(", ");
6193    if available.is_empty() {
6194        anyhow::bail!("inference router {router_id:?} is not registered");
6195    }
6196    anyhow::bail!(
6197        "inference router {router_id:?} is not registered; available routers: {available}"
6198    );
6199}
6200
6201fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6202    lookup_model(model)
6203        .map(|entry| {
6204            entry
6205                .supported_reasoning
6206                .iter()
6207                .any(|option| option.effort == effort)
6208        })
6209        .unwrap_or(false)
6210}
6211
6212fn is_final_answer_phase(phase: Option<&str>) -> bool {
6213    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6214}
6215
6216fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6217    cfg.model_edit_tools
6218        .get(model)
6219        .map(String::as_str)
6220        .or_else(|| {
6221            cfg.model_profiles
6222                .get(model)
6223                .and_then(|profile| profile.edit_tool.as_deref())
6224        })
6225        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6226        .or(Some(EDIT_TOOL_EDIT))
6227}
6228
6229fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6230    cfg.model_profiles
6231        .get(model)
6232        .cloned()
6233        .or_else(|| built_in_model_profile(model))
6234}
6235
6236pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6237    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6238}
6239
6240/// Provider-aware profile resolution for the active turn.
6241///
6242/// Many model ids are shared across providers (for example Cursor proxies
6243/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
6244/// first catalog entry, which assigns cross-provider ids the wrong family and
6245/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
6246/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
6247/// catalog entry, keeping user-configured profile overrides as the top
6248/// precedence.
6249fn model_profile_for_provider_model(
6250    cfg: &RuntimeConfig,
6251    provider: &str,
6252    model: &str,
6253) -> Option<ModelHarnessProfile> {
6254    cfg.model_profiles
6255        .get(model)
6256        .cloned()
6257        .or_else(|| built_in_model_profile_for_provider(provider, model))
6258}
6259
6260fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6261    profile
6262        .map(|profile| profile.schema_policy)
6263        .unwrap_or_default()
6264}
6265
6266fn model_profile_segment_metadata(
6267    profile: Option<&ModelHarnessProfile>,
6268    provider: &str,
6269    model: &str,
6270    segment: &str,
6271) -> serde_json::Value {
6272    serde_json::json!({
6273        "kind": MODEL_PROFILE_TRACE_KIND,
6274        "segment": segment,
6275        "provider": provider,
6276        "model": model,
6277        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6278        "providerFamily": profile.map(|profile| profile.provider_family),
6279        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6280        "schemaPolicy": profile.map(|profile| profile.schema_policy),
6281        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6282        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6283        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6284    })
6285}
6286
6287fn model_switch_summary(
6288    transcript: &[TranscriptItem],
6289    profile: Option<&ModelHarnessProfile>,
6290    provider: &str,
6291    model: &str,
6292    tools: &[roder_api::tools::ToolSpec],
6293) -> Option<String> {
6294    let previous = latest_model_profile_segment(transcript)?;
6295    let previous_model = previous
6296        .get("model")
6297        .and_then(serde_json::Value::as_str)
6298        .unwrap_or_default();
6299    let previous_provider = previous
6300        .get("provider")
6301        .and_then(serde_json::Value::as_str)
6302        .unwrap_or_default();
6303    if previous_model == model && previous_provider == provider {
6304        return None;
6305    }
6306
6307    let previous_profile = previous
6308        .get("profileModel")
6309        .and_then(serde_json::Value::as_str)
6310        .unwrap_or(previous_model);
6311    let current_profile = profile
6312        .map(|profile| profile.model.as_str())
6313        .unwrap_or(model);
6314    let previous_edit_tool = previous
6315        .get("editTool")
6316        .and_then(serde_json::Value::as_str)
6317        .unwrap_or("none");
6318    let current_edit_tool = profile
6319        .and_then(|profile| profile.edit_tool.as_deref())
6320        .unwrap_or("none");
6321    let tool_names = tools
6322        .iter()
6323        .map(|tool| tool.name.as_str())
6324        .take(12)
6325        .collect::<Vec<_>>()
6326        .join(", ");
6327    Some(format!(
6328        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6329        if tool_names.is_empty() {
6330            "none"
6331        } else {
6332            &tool_names
6333        }
6334    ))
6335}
6336
6337fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6338    transcript.iter().rev().find_map(|item| {
6339        let TranscriptItem::ProviderMetadata(value) = item else {
6340            return None;
6341        };
6342        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6343            .then_some(value)
6344    })
6345}
6346
6347pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6348    match value.trim() {
6349        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6350        _ => anyhow::bail!(
6351            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6352        ),
6353    }
6354}
6355
6356fn should_persist_thread_event(thread_id: &str) -> bool {
6357    !is_synthetic_event_thread_id(thread_id)
6358}
6359
6360#[cfg(test)]
6361#[path = "runtime/codex_v2_tests.rs"]
6362mod codex_v2_tests;
6363
6364#[cfg(test)]
6365#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6366mod codex_v2_lifecycle_tests;
6367
6368#[cfg(test)]
6369mod tests {
6370    use super::*;
6371    use futures::stream;
6372    use roder_api::catalog::{
6373        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6374        REASONING_NONE, REASONING_XHIGH,
6375    };
6376    use roder_api::extension::ExtensionRegistryBuilder;
6377    use roder_api::inference::{
6378        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6379        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6380        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6381        ReasoningEffortDescriptor,
6382    };
6383    use roder_api::inference_routing::{
6384        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6385    };
6386    use roder_api::thread::ThreadStoreFactory;
6387    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6388    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6389    use std::sync::Mutex as StdMutex;
6390
6391    fn test_workspace() -> String {
6392        std::env::current_dir().unwrap().display().to_string()
6393    }
6394
6395    #[test]
6396    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6397        let config = RuntimeConfig {
6398            runtime_profile: RuntimeProfile::Interactive,
6399            turn_deadline_seconds: Some(60),
6400            ..RuntimeConfig::default()
6401        };
6402
6403        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6404        let remaining = deadline_remaining_seconds(Some(deadline));
6405
6406        assert!(
6407            matches!(remaining, Some(1..=60)),
6408            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6409        );
6410    }
6411
6412    #[test]
6413    fn interactive_without_turn_deadline_stays_unbounded() {
6414        let config = RuntimeConfig {
6415            runtime_profile: RuntimeProfile::Interactive,
6416            turn_deadline_seconds: None,
6417            ..RuntimeConfig::default()
6418        };
6419
6420        assert_eq!(turn_deadline_for_config(&config), None);
6421    }
6422
6423    struct MetadataMissingStore;
6424
6425    #[async_trait::async_trait]
6426    impl ThreadStore for MetadataMissingStore {
6427        fn id(&self) -> roder_api::thread::ThreadStoreId {
6428            "metadata-missing-store".to_string()
6429        }
6430
6431        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6432            Ok(metadata)
6433        }
6434
6435        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6436            Ok(Vec::new())
6437        }
6438
6439        async fn load_thread(
6440            &self,
6441            _thread_id: &ThreadId,
6442        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6443            Ok(Some(ThreadSnapshot {
6444                metadata: None,
6445                ..ThreadSnapshot::default()
6446            }))
6447        }
6448
6449        async fn append_event(
6450            &self,
6451            _thread_id: &ThreadId,
6452            _envelope: &EventEnvelope,
6453        ) -> anyhow::Result<()> {
6454            Ok(())
6455        }
6456    }
6457
6458    struct MetadataMissingStoreFactory;
6459
6460    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6461        fn id(&self) -> roder_api::thread::ThreadStoreId {
6462            "metadata-missing-store".to_string()
6463        }
6464
6465        fn create(&self) -> Arc<dyn ThreadStore> {
6466            Arc::new(MetadataMissingStore)
6467        }
6468    }
6469
6470    #[test]
6471    fn synthetic_app_server_events_are_not_thread_events() {
6472        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6473            assert!(!should_persist_thread_event(thread_id));
6474        }
6475        assert!(should_persist_thread_event("thread-discovery"));
6476        assert!(should_persist_thread_event("thread-plan"));
6477        assert!(should_persist_thread_event("thread-process"));
6478        assert!(should_persist_thread_event("thread-1"));
6479    }
6480
6481    #[test]
6482    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6483        assert_eq!(
6484            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6485            Some(945_000)
6486        );
6487        assert_eq!(
6488            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6489            Some(115_200)
6490        );
6491    }
6492
6493    #[test]
6494    fn server_side_compaction_respects_explicit_config_override() {
6495        let cfg = RuntimeConfig {
6496            auto_compact_token_limit: Some(123_456),
6497            ..RuntimeConfig::default()
6498        };
6499
6500        assert_eq!(
6501            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6502            Some(123_456)
6503        );
6504    }
6505
6506    #[tokio::test]
6507    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6508        let captured = Arc::new(StdMutex::new(None));
6509        let mut builder = ExtensionRegistryBuilder::new();
6510        builder.inference_engine(Arc::new(CapturingEngine {
6511            request: captured.clone(),
6512        }));
6513        let thread_root = std::env::temp_dir().join(format!(
6514            "roder-pre-request-compaction-{}",
6515            uuid::Uuid::new_v4()
6516        ));
6517        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6518            base_path: thread_root.clone(),
6519        }));
6520        let runtime = Arc::new(
6521            Runtime::new(
6522                builder.build().unwrap(),
6523                RuntimeConfig {
6524                    default_provider: PROVIDER_MOCK.to_string(),
6525                    default_model: "gpt-5.5".to_string(),
6526                    file_backed_dynamic_context: true,
6527                    ..RuntimeConfig::default()
6528                },
6529            )
6530            .unwrap(),
6531        );
6532        let thread_id = runtime
6533            .create_thread(Some("Pre-request compaction".to_string()))
6534            .await
6535            .unwrap()
6536            .thread_id;
6537        let old_turn = "old-turn".to_string();
6538        runtime
6539            .persist_turn_item(
6540                &thread_id,
6541                &old_turn,
6542                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6543            )
6544            .await
6545            .unwrap();
6546
6547        let mut events = runtime.subscribe_events();
6548        runtime
6549            .start_turn(StartTurnRequest {
6550                thread_id: thread_id.clone(),
6551                message: "continue".to_string(),
6552                images: Vec::new(),
6553                provider_override: None,
6554                model_override: None,
6555                reasoning_override: None,
6556                workspace: test_workspace(),
6557                instructions: InstructionBundle::default(),
6558                developer_context: None,
6559                task_ledger_required: false,
6560            })
6561            .await
6562            .unwrap();
6563        loop {
6564            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6565                .await
6566                .unwrap()
6567                .unwrap();
6568            if envelope.thread_id.as_deref() == Some(&thread_id)
6569                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6570            {
6571                break;
6572            }
6573        }
6574
6575        let request = captured.lock().unwrap().clone().unwrap();
6576        assert!(
6577            matches!(
6578                request.transcript.first(),
6579                Some(TranscriptItem::ContextCompaction(_))
6580            ),
6581            "provider request should start with a local emergency compaction item"
6582        );
6583        assert!(
6584            request.transcript.len() < 4,
6585            "provider request should not replay the full oversized prior transcript: {:?}",
6586            request.transcript
6587        );
6588
6589        let _ = std::fs::remove_dir_all(thread_root);
6590    }
6591
6592    #[tokio::test]
6593    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6594        let captured = Arc::new(StdMutex::new(None));
6595        let mut builder = ExtensionRegistryBuilder::new();
6596        builder.inference_engine(Arc::new(CapturingEngine {
6597            request: captured.clone(),
6598        }));
6599        let thread_root = std::env::temp_dir().join(format!(
6600            "roder-context-failure-continue-{}",
6601            uuid::Uuid::new_v4()
6602        ));
6603        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6604            base_path: thread_root.clone(),
6605        }));
6606        let runtime = Arc::new(
6607            Runtime::new(
6608                builder.build().unwrap(),
6609                RuntimeConfig {
6610                    default_provider: PROVIDER_MOCK.to_string(),
6611                    default_model: "gpt-5.5".to_string(),
6612                    file_backed_dynamic_context: true,
6613                    ..RuntimeConfig::default()
6614                },
6615            )
6616            .unwrap(),
6617        );
6618        let thread_id = runtime
6619            .create_thread(Some("Context failure continue".to_string()))
6620            .await
6621            .unwrap()
6622            .thread_id;
6623        let failed_turn = "failed-turn".to_string();
6624        runtime
6625            .persist_turn_item(
6626                &thread_id,
6627                &failed_turn,
6628                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6629            )
6630            .await
6631            .unwrap();
6632        runtime
6633            .persist_turn_item(
6634                &thread_id,
6635                &failed_turn,
6636                &TranscriptItem::Error(ErrorRecord {
6637                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6638                        .to_string(),
6639                }),
6640            )
6641            .await
6642            .unwrap();
6643
6644        let mut events = runtime.subscribe_events();
6645        runtime
6646            .start_turn(StartTurnRequest {
6647                thread_id: thread_id.clone(),
6648                message: "continue".to_string(),
6649                images: Vec::new(),
6650                provider_override: None,
6651                model_override: None,
6652                reasoning_override: None,
6653                workspace: test_workspace(),
6654                instructions: InstructionBundle::default(),
6655                developer_context: None,
6656                task_ledger_required: false,
6657            })
6658            .await
6659            .unwrap();
6660        loop {
6661            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6662                .await
6663                .unwrap()
6664                .unwrap();
6665            if envelope.thread_id.as_deref() == Some(&thread_id)
6666                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6667            {
6668                break;
6669            }
6670        }
6671
6672        let request = captured.lock().unwrap().clone().unwrap();
6673        assert!(
6674            matches!(
6675                request.transcript.first(),
6676                Some(TranscriptItem::ContextCompaction(_))
6677            ),
6678            "provider request after context-window failure should start with local compaction"
6679        );
6680        assert!(
6681            request
6682                .transcript
6683                .iter()
6684                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6685            "current continue prompt must be preserved: {:?}",
6686            request.transcript
6687        );
6688        assert!(
6689            !request.transcript.iter().any(
6690                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6691            ),
6692            "raw prior context-window error should be summarized, not replayed: {:?}",
6693            request.transcript
6694        );
6695
6696        let _ = std::fs::remove_dir_all(thread_root);
6697    }
6698
6699    #[tokio::test]
6700    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6701        let workspace = test_workspace();
6702        let mut builder = ExtensionRegistryBuilder::new();
6703        builder.inference_engine(Arc::new(FakeInferenceEngine));
6704        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6705        let runtime = Runtime::new(
6706            builder.build().unwrap(),
6707            RuntimeConfig {
6708                workspace: Some(workspace.clone()),
6709                ..RuntimeConfig::default()
6710            },
6711        )
6712        .unwrap();
6713
6714        let resolved = runtime
6715            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6716            .await
6717            .unwrap();
6718
6719        assert_eq!(resolved, workspace);
6720    }
6721
6722    #[tokio::test]
6723    async fn automations_can_create_project_thread_with_model_overrides() {
6724        let runtime = Runtime::fake().unwrap();
6725        let workspace = std::env::temp_dir().join("project");
6726        let metadata = runtime
6727            .create_thread_with(CreateThreadRequest {
6728                title: Some("Automation: nightly status".to_string()),
6729                workspace: workspace.display().to_string(),
6730                workspace_id: None,
6731                root_id: None,
6732                provider: Some("mock".to_string()),
6733                model: Some("mock".to_string()),
6734                selection_mode: None,
6735                tool_allowlist: Vec::new(),
6736                developer_instructions: None,
6737                external_tools: Vec::new(),
6738                runner: None,
6739            })
6740            .await
6741            .unwrap();
6742
6743        assert_eq!(
6744            metadata.title.as_deref(),
6745            Some("Automation: nightly status")
6746        );
6747        assert_eq!(metadata.workspace, workspace.display().to_string());
6748        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6749        assert_eq!(metadata.model.as_deref(), Some("mock"));
6750    }
6751
6752    #[tokio::test]
6753    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6754        let captured = Arc::new(StdMutex::new(None));
6755        let mut builder = ExtensionRegistryBuilder::new();
6756        builder.inference_engine(Arc::new(CapturingEngine {
6757            request: captured.clone(),
6758        }));
6759        let thread_root = std::env::temp_dir().join(format!(
6760            "roder-provider-compaction-boundary-{}",
6761            uuid::Uuid::new_v4()
6762        ));
6763        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6764            base_path: thread_root.clone(),
6765        }));
6766        let runtime = Arc::new(
6767            Runtime::new(
6768                builder.build().unwrap(),
6769                RuntimeConfig {
6770                    default_provider: PROVIDER_MOCK.to_string(),
6771                    default_model: "gpt-5.5".to_string(),
6772                    ..RuntimeConfig::default()
6773                },
6774            )
6775            .unwrap(),
6776        );
6777        let thread_id = runtime
6778            .create_thread(Some("Provider compaction boundary".to_string()))
6779            .await
6780            .unwrap()
6781            .thread_id;
6782        let old_turn = "old-turn".to_string();
6783        runtime
6784            .persist_turn_item(
6785                &thread_id,
6786                &old_turn,
6787                &TranscriptItem::UserMessage(UserMessage::text(
6788                    "old history that must not be replayed after provider compaction",
6789                )),
6790            )
6791            .await
6792            .unwrap();
6793        runtime
6794            .persist_turn_item(
6795                &thread_id,
6796                &old_turn,
6797                &TranscriptItem::AssistantMessage(AssistantMessage {
6798                    text: "old answer".to_string(),
6799                    phase: None,
6800                }),
6801            )
6802            .await
6803            .unwrap();
6804        runtime
6805            .persist_turn_item(
6806                &thread_id,
6807                &old_turn,
6808                &TranscriptItem::ProviderMetadata(serde_json::json!({
6809                    "output": [{
6810                        "id": "cmp_1",
6811                        "type": "compaction",
6812                        "encrypted_content": "opaque-state"
6813                    }]
6814                })),
6815            )
6816            .await
6817            .unwrap();
6818        runtime
6819            .persist_turn_item(
6820                &thread_id,
6821                &old_turn,
6822                &TranscriptItem::AssistantMessage(AssistantMessage {
6823                    text: "after compact".to_string(),
6824                    phase: None,
6825                }),
6826            )
6827            .await
6828            .unwrap();
6829
6830        let mut events = runtime.subscribe_events();
6831        runtime
6832            .start_turn(StartTurnRequest {
6833                thread_id: thread_id.clone(),
6834                message: "continue".to_string(),
6835                images: Vec::new(),
6836                provider_override: None,
6837                model_override: None,
6838                reasoning_override: None,
6839                workspace: test_workspace(),
6840                instructions: InstructionBundle::default(),
6841                developer_context: None,
6842                task_ledger_required: false,
6843            })
6844            .await
6845            .unwrap();
6846        loop {
6847            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6848                .await
6849                .unwrap()
6850                .unwrap();
6851            if envelope.thread_id.as_deref() == Some(&thread_id)
6852                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6853            {
6854                break;
6855            }
6856        }
6857
6858        let request = captured.lock().unwrap().clone().unwrap();
6859        let compaction_idx = request.transcript.iter().position(|item| {
6860            matches!(
6861                item,
6862                TranscriptItem::ProviderMetadata(metadata)
6863                    if crate::compaction::provider_metadata_has_compaction(metadata)
6864            )
6865        });
6866        assert!(
6867            compaction_idx.is_some(),
6868            "next provider request should include the provider compaction item: {:?}",
6869            request.transcript
6870        );
6871        // Skill/context injectors may prepend non-history items, but no prior-turn
6872        // conversation may appear before the latest provider compaction boundary.
6873        let history_before_boundary =
6874            request
6875                .transcript
6876                .iter()
6877                .take(compaction_idx.unwrap())
6878                .any(|item| match item {
6879                    TranscriptItem::AssistantMessage(_)
6880                    | TranscriptItem::ToolCall(_)
6881                    | TranscriptItem::ToolResult(_) => true,
6882                    TranscriptItem::UserMessage(message) => {
6883                        !message.text.contains("<skills>") && message.text != "continue"
6884                    }
6885                    _ => false,
6886                });
6887        assert!(
6888            !history_before_boundary,
6889            "pre-provider-compaction conversation must not be replayed: {:?}",
6890            request.transcript
6891        );
6892        assert!(
6893            !request.transcript.iter().any(|item| {
6894                matches!(
6895                    item,
6896                    TranscriptItem::UserMessage(message)
6897                        if message.text.contains("old history that must not be replayed")
6898                )
6899            }),
6900            "pre-provider-compaction history must not be replayed: {:?}",
6901            request.transcript
6902        );
6903        assert!(
6904            request.transcript.iter().any(|item| {
6905                matches!(
6906                    item,
6907                    TranscriptItem::UserMessage(message) if message.text == "continue"
6908                )
6909            }),
6910            "current continue prompt must be preserved: {:?}",
6911            request.transcript
6912        );
6913        assert_eq!(
6914            request.runtime.auto_compact_token_limit,
6915            Some(945_000),
6916            "gpt-5.5 should keep server-side auto compaction configured"
6917        );
6918
6919        let _ = std::fs::remove_dir_all(thread_root);
6920    }
6921
6922    #[test]
6923    fn server_side_compaction_is_only_enabled_for_supported_models() {
6924        let cfg = RuntimeConfig {
6925            auto_compact_token_limit: Some(123_456),
6926            ..RuntimeConfig::default()
6927        };
6928
6929        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6930        assert_eq!(
6931            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6932            None
6933        );
6934    }
6935
6936    #[test]
6937    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6938        let cfg = RuntimeConfig {
6939            reasoning: Some(REASONING_HIGH.to_string()),
6940            ..RuntimeConfig::default()
6941        };
6942
6943        assert_eq!(
6944            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6945            REASONING_NONE
6946        );
6947        assert_eq!(
6948            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6949            ReasoningConfig::default()
6950        );
6951    }
6952
6953    #[test]
6954    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6955        let cfg = RuntimeConfig {
6956            reasoning: Some(REASONING_MINIMAL.to_string()),
6957            ..RuntimeConfig::default()
6958        };
6959
6960        assert_eq!(
6961            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6962            REASONING_MEDIUM
6963        );
6964    }
6965
6966    #[test]
6967    fn unsupported_configured_gemini_reasoning_is_rejected() {
6968        let mut builder = ExtensionRegistryBuilder::new();
6969        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6970
6971        let err = match Runtime::new(
6972            builder.build().unwrap(),
6973            RuntimeConfig {
6974                default_model: "gemini-3.5-flash".to_string(),
6975                reasoning: Some(REASONING_XHIGH.to_string()),
6976                ..RuntimeConfig::default()
6977            },
6978        ) {
6979            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6980            Err(err) => err,
6981        };
6982
6983        assert!(
6984            err.to_string()
6985                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6986        );
6987    }
6988
6989    #[tokio::test]
6990    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6991        let runtime = Runtime::new(
6992            Runtime::fake().unwrap().registry,
6993            RuntimeConfig {
6994                reasoning: Some(REASONING_HIGH.to_string()),
6995                ..RuntimeConfig::default()
6996            },
6997        )
6998        .unwrap();
6999
7000        let cfg = runtime
7001            .select_provider(
7002                roder_api::catalog::PROVIDER_MOCK.to_string(),
7003                Some("claude-haiku-4-5-20251001".to_string()),
7004                Some(REASONING_NONE.to_string()),
7005            )
7006            .await
7007            .unwrap();
7008
7009        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7010        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7011    }
7012
7013    #[tokio::test]
7014    async fn selecting_none_for_model_that_supports_none_updates_preference() {
7015        let runtime = Runtime::new(
7016            Runtime::fake().unwrap().registry,
7017            RuntimeConfig {
7018                reasoning: Some(REASONING_HIGH.to_string()),
7019                ..RuntimeConfig::default()
7020            },
7021        )
7022        .unwrap();
7023
7024        let cfg = runtime
7025            .select_provider(
7026                roder_api::catalog::PROVIDER_MOCK.to_string(),
7027                Some("mock".to_string()),
7028                Some(REASONING_NONE.to_string()),
7029            )
7030            .await
7031            .unwrap();
7032
7033        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7034    }
7035
7036    #[test]
7037    fn parallel_tool_calls_default_on_with_model_override() {
7038        assert!(parallel_tool_calls_for_model(
7039            &RuntimeConfig::default(),
7040            "custom-model"
7041        ));
7042
7043        let cfg = RuntimeConfig {
7044            model_parallel_tool_calls: std::collections::HashMap::from([(
7045                "custom-model".to_string(),
7046                false,
7047            )]),
7048            ..RuntimeConfig::default()
7049        };
7050
7051        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7052        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7053    }
7054
7055    #[test]
7056    fn profile_parallel_tool_calls_applies_between_config_and_default() {
7057        let cfg = RuntimeConfig {
7058            model_profiles: std::collections::HashMap::from([(
7059                "gpt-5.5".to_string(),
7060                test_model_profile("gpt-5.5"),
7061            )]),
7062            ..RuntimeConfig::default()
7063        };
7064
7065        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7066
7067        let cfg = RuntimeConfig {
7068            model_parallel_tool_calls: std::collections::HashMap::from([(
7069                "gpt-5.5".to_string(),
7070                true,
7071            )]),
7072            ..cfg
7073        };
7074
7075        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7076    }
7077
7078    struct CapturingEngine {
7079        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7080    }
7081
7082    #[async_trait::async_trait]
7083    impl InferenceEngine for CapturingEngine {
7084        fn id(&self) -> String {
7085            roder_api::catalog::PROVIDER_MOCK.to_string()
7086        }
7087
7088        fn capabilities(&self) -> InferenceCapabilities {
7089            InferenceCapabilities::coding_agent_default()
7090        }
7091
7092        async fn list_models(
7093            &self,
7094            _ctx: InferenceProviderContext<'_>,
7095        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7096            Ok(roder_api::catalog::models_for_provider(
7097                roder_api::catalog::PROVIDER_MOCK,
7098                true,
7099            ))
7100        }
7101
7102        async fn stream_turn(
7103            &self,
7104            _ctx: InferenceTurnContext<'_>,
7105            request: AgentInferenceRequest,
7106        ) -> anyhow::Result<InferenceEventStream> {
7107            *self.request.lock().unwrap() = Some(request);
7108            Ok(Box::pin(stream::iter(vec![
7109                Ok(InferenceEvent::MessageDelta(MessageDelta {
7110                    text: "done".to_string(),
7111                    phase: None,
7112                })),
7113                Ok(InferenceEvent::Completed(CompletionMetadata {
7114                    stop_reason: Some("stop".to_string()),
7115                    provider_response_id: None,
7116                })),
7117            ])))
7118        }
7119    }
7120
7121    struct RoutingCaptureEngine {
7122        id: &'static str,
7123        models: Vec<ModelDescriptor>,
7124        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7125    }
7126
7127    #[async_trait::async_trait]
7128    impl InferenceEngine for RoutingCaptureEngine {
7129        fn id(&self) -> String {
7130            self.id.to_string()
7131        }
7132
7133        fn capabilities(&self) -> InferenceCapabilities {
7134            InferenceCapabilities::coding_agent_default()
7135        }
7136
7137        async fn list_models(
7138            &self,
7139            _ctx: InferenceProviderContext<'_>,
7140        ) -> anyhow::Result<Vec<ModelDescriptor>> {
7141            Ok(self.models.clone())
7142        }
7143
7144        async fn stream_turn(
7145            &self,
7146            _ctx: InferenceTurnContext<'_>,
7147            request: AgentInferenceRequest,
7148        ) -> anyhow::Result<InferenceEventStream> {
7149            self.requests.lock().unwrap().push(request);
7150            Ok(Box::pin(stream::iter(vec![
7151                Ok(InferenceEvent::MessageDelta(MessageDelta {
7152                    text: "routed".to_string(),
7153                    phase: None,
7154                })),
7155                Ok(InferenceEvent::Completed(CompletionMetadata {
7156                    stop_reason: Some("stop".to_string()),
7157                    provider_response_id: None,
7158                })),
7159            ])))
7160        }
7161    }
7162
7163    struct StaticRouter {
7164        id: &'static str,
7165        decision: InferenceRoutingDecision,
7166        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7167    }
7168
7169    #[async_trait::async_trait]
7170    impl InferenceRouter for StaticRouter {
7171        fn id(&self) -> String {
7172            self.id.to_string()
7173        }
7174
7175        async fn route(
7176            &self,
7177            context: InferenceRoutingContext,
7178        ) -> anyhow::Result<InferenceRoutingDecision> {
7179            self.contexts.lock().unwrap().push(context);
7180            Ok(self.decision.clone())
7181        }
7182    }
7183
7184    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7185        ModelDescriptor {
7186            id: id.to_string(),
7187            name: id.to_string(),
7188            context_window: Some(128_000),
7189            default_reasoning: supported_reasoning
7190                .first()
7191                .map(|effort| (*effort).to_string()),
7192            supported_reasoning: supported_reasoning
7193                .iter()
7194                .map(|effort| ReasoningEffortDescriptor {
7195                    effort: (*effort).to_string(),
7196                    description: format!("{effort} reasoning"),
7197                })
7198                .collect(),
7199        }
7200    }
7201
7202    struct TaskLedgerCompletionGateEngine {
7203        calls: StdMutex<u32>,
7204        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7205    }
7206
7207    #[async_trait::async_trait]
7208    impl InferenceEngine for TaskLedgerCompletionGateEngine {
7209        fn id(&self) -> String {
7210            roder_api::catalog::PROVIDER_MOCK.to_string()
7211        }
7212
7213        fn capabilities(&self) -> InferenceCapabilities {
7214            InferenceCapabilities::coding_agent_default()
7215        }
7216
7217        async fn list_models(
7218            &self,
7219            _ctx: InferenceProviderContext<'_>,
7220        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7221            Ok(roder_api::catalog::models_for_provider(
7222                roder_api::catalog::PROVIDER_MOCK,
7223                true,
7224            ))
7225        }
7226
7227        async fn stream_turn(
7228            &self,
7229            _ctx: InferenceTurnContext<'_>,
7230            request: AgentInferenceRequest,
7231        ) -> anyhow::Result<InferenceEventStream> {
7232            self.requests.lock().unwrap().push(request);
7233            let mut calls = self.calls.lock().unwrap();
7234            *calls += 1;
7235            let events = match *calls {
7236                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7237                    id: "ledger-open".to_string(),
7238                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7239                    arguments: serde_json::json!({
7240                        "tasks": [
7241                            {
7242                                "id": "inspect",
7243                                "content": "Inspect local assets",
7244                                "status": "completed",
7245                                "evidence": "listed workspace"
7246                            },
7247                            {
7248                                "id": "write",
7249                                "content": "Write /app/result.txt",
7250                                "status": "pending"
7251                            }
7252                        ],
7253                        "requireCompletionEvidence": true
7254                    })
7255                    .to_string(),
7256                }))],
7257                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7258                    id: "ledger-complete".to_string(),
7259                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7260                    arguments: serde_json::json!({
7261                        "tasks": [
7262                            {
7263                                "id": "inspect",
7264                                "content": "Inspect local assets",
7265                                "status": "completed",
7266                                "evidence": "listed workspace"
7267                            },
7268                            {
7269                                "id": "write",
7270                                "content": "Write /app/result.txt",
7271                                "status": "completed",
7272                                "evidence": "wrote answer"
7273                            }
7274                        ],
7275                        "requireCompletionEvidence": true
7276                    })
7277                    .to_string(),
7278                }))],
7279                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7280                    text: "final".to_string(),
7281                    phase: None,
7282                }))],
7283            };
7284            Ok(Box::pin(stream::iter(events.into_iter().chain(
7285                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7286                    stop_reason: Some("stop".to_string()),
7287                    provider_response_id: None,
7288                }))),
7289            ))))
7290        }
7291    }
7292
7293    struct VerificationGateEngine {
7294        calls: StdMutex<u32>,
7295    }
7296
7297    #[async_trait::async_trait]
7298    impl InferenceEngine for VerificationGateEngine {
7299        fn id(&self) -> String {
7300            roder_api::catalog::PROVIDER_MOCK.to_string()
7301        }
7302
7303        fn capabilities(&self) -> InferenceCapabilities {
7304            InferenceCapabilities::coding_agent_default()
7305        }
7306
7307        async fn list_models(
7308            &self,
7309            _ctx: InferenceProviderContext<'_>,
7310        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7311            Ok(roder_api::catalog::models_for_provider(
7312                roder_api::catalog::PROVIDER_MOCK,
7313                true,
7314            ))
7315        }
7316
7317        async fn stream_turn(
7318            &self,
7319            _ctx: InferenceTurnContext<'_>,
7320            request: AgentInferenceRequest,
7321        ) -> anyhow::Result<InferenceEventStream> {
7322            let mut calls = self.calls.lock().unwrap();
7323            *calls += 1;
7324            let events = match *calls {
7325                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7326                    id: "write-1".to_string(),
7327                    name: "write_file".to_string(),
7328                    arguments: serde_json::json!({
7329                        "path": "src/lib.rs",
7330                        "content": "pub fn answer() -> u8 { 42 }\n"
7331                    })
7332                    .to_string(),
7333                }))],
7334                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7335                    text: "done too early".to_string(),
7336                    phase: None,
7337                }))],
7338                3 if request.transcript.iter().any(|item| {
7339                    matches!(
7340                        item,
7341                        TranscriptItem::UserMessage(message)
7342                            if message.text.contains("Verification gate blocked final completion")
7343                    )
7344                }) =>
7345                {
7346                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7347                        id: "verify-1".to_string(),
7348                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7349                        arguments: serde_json::json!({
7350                            "originalTask": "write code",
7351                            "changedFiles": ["src/lib.rs"],
7352                            "toolEvidence": ["write_file wrote src/lib.rs"],
7353                            "testsRun": ["cargo test -p roder-core verification_gate"],
7354                            "openGaps": [],
7355                            "status": "completed"
7356                        })
7357                        .to_string(),
7358                    }))]
7359                }
7360                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7361                    text: "verified final".to_string(),
7362                    phase: None,
7363                }))],
7364            };
7365            Ok(Box::pin(stream::iter(events.into_iter().chain(
7366                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7367                    stop_reason: Some("stop".to_string()),
7368                    provider_response_id: None,
7369                }))),
7370            ))))
7371        }
7372    }
7373
7374    struct SpeedPolicyEngine {
7375        calls: StdMutex<u32>,
7376        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7377    }
7378
7379    #[async_trait::async_trait]
7380    impl InferenceEngine for SpeedPolicyEngine {
7381        fn id(&self) -> String {
7382            roder_api::catalog::PROVIDER_MOCK.to_string()
7383        }
7384
7385        fn capabilities(&self) -> InferenceCapabilities {
7386            InferenceCapabilities::coding_agent_default()
7387        }
7388
7389        async fn list_models(
7390            &self,
7391            _ctx: InferenceProviderContext<'_>,
7392        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7393            Ok(roder_api::catalog::models_for_provider(
7394                roder_api::catalog::PROVIDER_MOCK,
7395                true,
7396            ))
7397        }
7398
7399        async fn stream_turn(
7400            &self,
7401            _ctx: InferenceTurnContext<'_>,
7402            request: AgentInferenceRequest,
7403        ) -> anyhow::Result<InferenceEventStream> {
7404            self.requests.lock().unwrap().push(request.clone());
7405            let mut calls = self.calls.lock().unwrap();
7406            *calls += 1;
7407            let events = match *calls {
7408                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7409                    id: "write-1".to_string(),
7410                    name: "write_file".to_string(),
7411                    arguments: serde_json::json!({
7412                        "path": "src/lib.rs",
7413                        "content": "pub fn answer() -> u8 { 42 }\n"
7414                    })
7415                    .to_string(),
7416                }))],
7417                3 if request.transcript.iter().any(|item| {
7418                    matches!(
7419                        item,
7420                        TranscriptItem::UserMessage(message)
7421                            if message.text.contains("Verification gate blocked final completion")
7422                    )
7423                }) =>
7424                {
7425                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7426                        id: "verify-1".to_string(),
7427                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7428                        arguments: serde_json::json!({
7429                            "originalTask": "write code",
7430                            "changedFiles": ["src/lib.rs"],
7431                            "toolEvidence": ["write_file wrote src/lib.rs"],
7432                            "testsRun": ["cargo test -p roder-core speed_policy"],
7433                            "openGaps": [],
7434                            "status": "completed"
7435                        })
7436                        .to_string(),
7437                    }))]
7438                }
7439                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7440                    text: "done".to_string(),
7441                    phase: None,
7442                }))],
7443            };
7444            Ok(Box::pin(stream::iter(events.into_iter().chain(
7445                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7446                    stop_reason: Some("stop".to_string()),
7447                    provider_response_id: None,
7448                }))),
7449            ))))
7450        }
7451    }
7452
7453    struct SwitchCaptureEngine {
7454        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7455    }
7456
7457    #[async_trait::async_trait]
7458    impl InferenceEngine for SwitchCaptureEngine {
7459        fn id(&self) -> String {
7460            roder_api::catalog::PROVIDER_MOCK.to_string()
7461        }
7462
7463        fn capabilities(&self) -> InferenceCapabilities {
7464            InferenceCapabilities::coding_agent_default()
7465        }
7466
7467        async fn list_models(
7468            &self,
7469            _ctx: InferenceProviderContext<'_>,
7470        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7471            Ok(roder_api::catalog::models_for_provider(
7472                roder_api::catalog::PROVIDER_MOCK,
7473                true,
7474            ))
7475        }
7476
7477        async fn stream_turn(
7478            &self,
7479            _ctx: InferenceTurnContext<'_>,
7480            request: AgentInferenceRequest,
7481        ) -> anyhow::Result<InferenceEventStream> {
7482            self.requests.lock().unwrap().push(request);
7483            Ok(Box::pin(stream::iter(vec![
7484                Ok(InferenceEvent::MessageDelta(MessageDelta {
7485                    text: "done".to_string(),
7486                    phase: None,
7487                })),
7488                Ok(InferenceEvent::Completed(CompletionMetadata {
7489                    stop_reason: Some("stop".to_string()),
7490                    provider_response_id: None,
7491                })),
7492            ])))
7493        }
7494    }
7495
7496    struct DeadlineEngine;
7497
7498    #[async_trait::async_trait]
7499    impl InferenceEngine for DeadlineEngine {
7500        fn id(&self) -> String {
7501            roder_api::catalog::PROVIDER_MOCK.to_string()
7502        }
7503
7504        fn capabilities(&self) -> InferenceCapabilities {
7505            InferenceCapabilities::coding_agent_default()
7506        }
7507
7508        async fn list_models(
7509            &self,
7510            _ctx: InferenceProviderContext<'_>,
7511        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7512            Ok(Vec::new())
7513        }
7514
7515        async fn stream_turn(
7516            &self,
7517            _ctx: InferenceTurnContext<'_>,
7518            _request: AgentInferenceRequest,
7519        ) -> anyhow::Result<InferenceEventStream> {
7520            Ok(Box::pin(stream::once(async {
7521                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7522                Ok(InferenceEvent::MessageDelta(MessageDelta {
7523                    text: "too late".to_string(),
7524                    phase: None,
7525                }))
7526            })))
7527        }
7528    }
7529
7530    struct WriteFileContributor;
7531
7532    impl ToolContributor for WriteFileContributor {
7533        fn id(&self) -> String {
7534            "test-write".to_string()
7535        }
7536
7537        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7538            registry.register(Arc::new(WriteFileTool))
7539        }
7540    }
7541
7542    struct WriteFileTool;
7543
7544    #[async_trait::async_trait]
7545    impl ToolExecutor for WriteFileTool {
7546        fn spec(&self) -> ToolSpec {
7547            ToolSpec {
7548                name: "write_file".to_string(),
7549                description: "Write a test file.".to_string(),
7550                parameters: serde_json::json!({
7551                    "type": "object",
7552                    "properties": {
7553                        "path": { "type": "string" },
7554                        "content": { "type": "string" }
7555                    },
7556                    "required": ["path", "content"],
7557                    "additionalProperties": false
7558                }),
7559            }
7560        }
7561
7562        async fn execute(
7563            &self,
7564            _ctx: ToolExecutionContext,
7565            call: ToolCall,
7566        ) -> anyhow::Result<ToolResult> {
7567            let path = call
7568                .arguments
7569                .get("path")
7570                .and_then(serde_json::Value::as_str)
7571                .unwrap_or("src/lib.rs");
7572            Ok(ToolResult {
7573                id: call.id,
7574                name: call.name,
7575                text: format!("wrote {path}"),
7576                data: serde_json::json!({ "path": path }),
7577                is_error: false,
7578            })
7579        }
7580    }
7581
7582    struct ProfileToolContributor;
7583
7584    impl ToolContributor for ProfileToolContributor {
7585        fn id(&self) -> String {
7586            "profile-tools".to_string()
7587        }
7588
7589        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7590            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7591                registry.register(Arc::new(ProfileTool {
7592                    name: name.to_string(),
7593                }))?;
7594            }
7595            Ok(())
7596        }
7597    }
7598
7599    struct ProfileTool {
7600        name: String,
7601    }
7602
7603    #[async_trait::async_trait]
7604    impl ToolExecutor for ProfileTool {
7605        fn spec(&self) -> ToolSpec {
7606            ToolSpec {
7607                name: self.name.clone(),
7608                description: format!("{} test tool", self.name),
7609                parameters: serde_json::json!({
7610                    "type": "object",
7611                    "properties": {
7612                        "path": { "type": "string" },
7613                        "content": { "type": "string" }
7614                    },
7615                    "required": ["path", "content"],
7616                    "additionalProperties": false
7617                }),
7618            }
7619        }
7620
7621        async fn execute(
7622            &self,
7623            _ctx: ToolExecutionContext,
7624            call: ToolCall,
7625        ) -> anyhow::Result<ToolResult> {
7626            Ok(ToolResult {
7627                id: call.id,
7628                name: call.name,
7629                text: "ok".to_string(),
7630                data: serde_json::json!({}),
7631                is_error: false,
7632            })
7633        }
7634    }
7635
7636    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7637        ModelHarnessProfile {
7638            model: model.to_string(),
7639            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7640            provider_family: ProviderFamily::OpenAi,
7641            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7642            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7643            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7644            reasoning: ModelProfileReasoning {
7645                orientation: Some(REASONING_LOW.to_string()),
7646                execution: Some(REASONING_LOW.to_string()),
7647                verification: Some(REASONING_LOW.to_string()),
7648                recovery: Some(REASONING_LOW.to_string()),
7649            },
7650            parallel_tool_calls: Some(false),
7651            auto_compact_token_limit: Some(123_000),
7652        }
7653    }
7654
7655    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7656        let captured = Arc::new(StdMutex::new(None));
7657        let mut builder = ExtensionRegistryBuilder::new();
7658        builder.inference_engine(Arc::new(CapturingEngine {
7659            request: captured.clone(),
7660        }));
7661        builder.tool_contributor(Arc::new(ProfileToolContributor));
7662        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7663        let mut rx = runtime.subscribe_events();
7664        let turn_id = runtime
7665            .start_turn(StartTurnRequest {
7666                thread_id: "thread-model-profile".to_string(),
7667                message: "use profile knobs".to_string(),
7668                images: Vec::new(),
7669                provider_override: None,
7670                model_override: None,
7671                reasoning_override: None,
7672                workspace: test_workspace(),
7673                instructions: InstructionBundle {
7674                    system: None,
7675                    developer: Some("base developer".to_string()),
7676                    developer_context: None,
7677                },
7678                developer_context: None,
7679                task_ledger_required: false,
7680            })
7681            .await
7682            .unwrap();
7683
7684        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7685            loop {
7686                let envelope = rx.recv().await.unwrap();
7687                if envelope.turn_id.as_deref() != Some(&turn_id) {
7688                    continue;
7689                }
7690                match envelope.event {
7691                    RoderEvent::TurnCompleted(_) => break,
7692                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7693                    _ => {}
7694                }
7695            }
7696        })
7697        .await
7698        .unwrap();
7699
7700        captured.lock().unwrap().clone().unwrap()
7701    }
7702
7703    struct ToolThenStopEngine {
7704        calls: StdMutex<u32>,
7705    }
7706
7707    #[async_trait::async_trait]
7708    impl InferenceEngine for ToolThenStopEngine {
7709        fn id(&self) -> String {
7710            roder_api::catalog::PROVIDER_MOCK.to_string()
7711        }
7712
7713        fn capabilities(&self) -> InferenceCapabilities {
7714            InferenceCapabilities::coding_agent_default()
7715        }
7716
7717        async fn list_models(
7718            &self,
7719            _ctx: InferenceProviderContext<'_>,
7720        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7721            Ok(roder_api::catalog::models_for_provider(
7722                roder_api::catalog::PROVIDER_MOCK,
7723                true,
7724            ))
7725        }
7726
7727        async fn stream_turn(
7728            &self,
7729            _ctx: InferenceTurnContext<'_>,
7730            _request: AgentInferenceRequest,
7731        ) -> anyhow::Result<InferenceEventStream> {
7732            let mut calls = self.calls.lock().unwrap();
7733            *calls += 1;
7734            let events = match *calls {
7735                1 => vec![
7736                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7737                        id: "write-1".to_string(),
7738                        name: "write_file".to_string(),
7739                        arguments: serde_json::json!({
7740                            "path": "src/lib.rs",
7741                            "content": "pub fn answer() -> u8 { 42 }\n"
7742                        })
7743                        .to_string(),
7744                    })),
7745                    Ok(InferenceEvent::Completed(CompletionMetadata {
7746                        stop_reason: Some("tool_use".to_string()),
7747                        provider_response_id: None,
7748                    })),
7749                ],
7750                _ => vec![
7751                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7752                        text: "final".to_string(),
7753                        phase: None,
7754                    })),
7755                    Ok(InferenceEvent::Completed(CompletionMetadata {
7756                        stop_reason: Some("end_turn".to_string()),
7757                        provider_response_id: None,
7758                    })),
7759                ],
7760            };
7761            Ok(Box::pin(stream::iter(events)))
7762        }
7763    }
7764
7765    #[tokio::test]
7766    async fn turn_completed_reports_terminal_step_finish_reason() {
7767        let mut builder = ExtensionRegistryBuilder::new();
7768        builder.inference_engine(Arc::new(ToolThenStopEngine {
7769            calls: StdMutex::new(0),
7770        }));
7771        builder.tool_contributor(Arc::new(WriteFileContributor));
7772        let runtime = Arc::new(
7773            Runtime::new(
7774                builder.build().unwrap(),
7775                RuntimeConfig {
7776                    policy_mode: PolicyMode::Bypass,
7777                    agent_swarm_mode: false,
7778                    ultra_mode: false,
7779                    ..RuntimeConfig::default()
7780                },
7781            )
7782            .unwrap(),
7783        );
7784        let mut rx = runtime.subscribe_events();
7785        let turn_id = runtime
7786            .start_turn(StartTurnRequest {
7787                thread_id: "thread-finish-reason".to_string(),
7788                message: "write then finish".to_string(),
7789                images: Vec::new(),
7790                provider_override: None,
7791                model_override: None,
7792                reasoning_override: None,
7793                workspace: test_workspace(),
7794                instructions: InstructionBundle {
7795                    system: None,
7796                    developer: None,
7797                    developer_context: None,
7798                },
7799                developer_context: None,
7800                task_ledger_required: false,
7801            })
7802            .await
7803            .unwrap();
7804
7805        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7806            loop {
7807                let envelope = rx.recv().await.unwrap();
7808                if envelope.turn_id.as_deref() != Some(&turn_id) {
7809                    continue;
7810                }
7811                match envelope.event {
7812                    RoderEvent::TurnCompleted(event) => break event,
7813                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7814                    _ => {}
7815                }
7816            }
7817        })
7818        .await
7819        .unwrap();
7820
7821        // The mid-turn tool_use stop reason must not leak; the terminal
7822        // end_turn step decides the turn's finish reason.
7823        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7824    }
7825
7826    #[tokio::test]
7827    async fn inference_router_selection_changes_request_model_and_records_event() {
7828        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7829        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7830        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7831        let selected = ModelSelection {
7832            provider: "routed-provider".to_string(),
7833            model: "routed-model".to_string(),
7834        };
7835        let default = ModelSelection {
7836            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7837            model: "mock".to_string(),
7838        };
7839        let decision = InferenceRoutingDecision {
7840            reasoning: Some(ReasoningConfig {
7841                enabled: true,
7842                level: Some(REASONING_LOW.to_string()),
7843            }),
7844            confidence: Some(0.91),
7845            baseline: Some(default.clone()),
7846            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7847                "intent", "routine",
7848            )],
7849            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7850        };
7851
7852        let mut builder = ExtensionRegistryBuilder::new();
7853        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7854            id: roder_api::catalog::PROVIDER_MOCK,
7855            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7856            requests: default_requests.clone(),
7857        }));
7858        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7859            id: "routed-provider",
7860            models: vec![routing_test_model(
7861                "routed-model",
7862                &[REASONING_LOW, REASONING_MEDIUM],
7863            )],
7864            requests: routed_requests.clone(),
7865        }));
7866        builder.inference_router(Arc::new(StaticRouter {
7867            id: "test-router",
7868            decision,
7869            contexts: contexts.clone(),
7870        }));
7871        let thread_root =
7872            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7873        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7874            base_path: thread_root.clone(),
7875        }));
7876        let runtime = Arc::new(
7877            Runtime::new(
7878                builder.build().unwrap(),
7879                RuntimeConfig {
7880                    default_provider: default.provider.clone(),
7881                    default_model: default.model.clone(),
7882                    ..RuntimeConfig::default()
7883                },
7884            )
7885            .unwrap(),
7886        );
7887        let thread_id = runtime
7888            .create_thread_with(CreateThreadRequest {
7889                title: Some("Routing auto".to_string()),
7890                workspace: test_workspace(),
7891                workspace_id: None,
7892                root_id: None,
7893                provider: Some(default.provider.clone()),
7894                model: Some(default.model.clone()),
7895                tool_allowlist: Vec::new(),
7896                developer_instructions: None,
7897                external_tools: Vec::new(),
7898                selection_mode: Some(ModelSelectionMode::auto(
7899                    "test-router:coding",
7900                    "test-router",
7901                    "Auto: Coding",
7902                    default.clone(),
7903                    Some("coding".to_string()),
7904                    None,
7905                )),
7906                runner: None,
7907            })
7908            .await
7909            .unwrap()
7910            .thread_id;
7911        let mut rx = runtime.subscribe_events();
7912        let turn_id = runtime
7913            .start_turn(StartTurnRequest {
7914                thread_id: thread_id.clone(),
7915                message: "small cleanup".to_string(),
7916                images: Vec::new(),
7917                provider_override: None,
7918                model_override: None,
7919                reasoning_override: None,
7920                workspace: test_workspace(),
7921                instructions: InstructionBundle::default(),
7922                developer_context: None,
7923                task_ledger_required: false,
7924            })
7925            .await
7926            .unwrap();
7927
7928        let mut routing_event = None;
7929        let mut inference_started = None;
7930        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7931            loop {
7932                let envelope = rx.recv().await.unwrap();
7933                if envelope.turn_id.as_deref() != Some(&turn_id) {
7934                    continue;
7935                }
7936                match envelope.event {
7937                    RoderEvent::InferenceRoutingDecision(event) => {
7938                        routing_event = Some(event);
7939                    }
7940                    RoderEvent::InferenceStarted(event) => {
7941                        inference_started = Some(event);
7942                    }
7943                    RoderEvent::TurnCompleted(_) => break,
7944                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7945                    _ => {}
7946                }
7947            }
7948        })
7949        .await
7950        .unwrap();
7951
7952        assert!(default_requests.lock().unwrap().is_empty());
7953        let routed_requests = routed_requests.lock().unwrap();
7954        assert_eq!(routed_requests.len(), 1);
7955        assert_eq!(routed_requests[0].model, selected);
7956        assert_eq!(
7957            routed_requests[0].reasoning.level.as_deref(),
7958            Some(REASONING_LOW)
7959        );
7960        assert_eq!(
7961            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7962            "selected"
7963        );
7964
7965        let routing_event = routing_event.expect("routing decision event");
7966        assert_eq!(routing_event.default_selection, default);
7967        assert_eq!(routing_event.selected_selection, selected);
7968        assert_eq!(
7969            routing_event.decision.outcome,
7970            InferenceRoutingOutcome::Selected
7971        );
7972        assert_eq!(
7973            inference_started.expect("inference started event").model,
7974            selected
7975        );
7976
7977        let contexts = contexts.lock().unwrap();
7978        assert_eq!(contexts.len(), 1);
7979        assert_eq!(contexts[0].default_selection, default);
7980        assert_eq!(contexts[0].candidates.len(), 2);
7981        assert!(
7982            contexts[0]
7983                .signals
7984                .iter()
7985                .any(|signal| signal.key == "profile" && signal.value == "coding")
7986        );
7987        let _ = std::fs::remove_dir_all(thread_root);
7988    }
7989
7990    #[tokio::test]
7991    async fn inference_router_is_bypassed_for_explicit_selection() {
7992        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7993        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7994        let selected = ModelSelection {
7995            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7996            model: "mock".to_string(),
7997        };
7998
7999        let mut builder = ExtensionRegistryBuilder::new();
8000        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8001            id: roder_api::catalog::PROVIDER_MOCK,
8002            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8003            requests: requests.clone(),
8004        }));
8005        builder.inference_router(Arc::new(StaticRouter {
8006            id: "test-router",
8007            decision: InferenceRoutingDecision::selected(
8008                "test-router",
8009                ModelSelection {
8010                    provider: "missing".to_string(),
8011                    model: "missing".to_string(),
8012                },
8013                "would route if called",
8014            ),
8015            contexts: contexts.clone(),
8016        }));
8017        let thread_root =
8018            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8019        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8020            base_path: thread_root.clone(),
8021        }));
8022        let runtime = Arc::new(
8023            Runtime::new(
8024                builder.build().unwrap(),
8025                RuntimeConfig {
8026                    default_provider: selected.provider.clone(),
8027                    default_model: selected.model.clone(),
8028                    inference_router: RuntimeInferenceRouterConfig {
8029                        enabled: true,
8030                        router_id: Some("test-router".to_string()),
8031                    },
8032                    ..RuntimeConfig::default()
8033                },
8034            )
8035            .unwrap(),
8036        );
8037        let thread_id = runtime
8038            .create_thread_with(CreateThreadRequest {
8039                title: Some("Routing explicit".to_string()),
8040                workspace: test_workspace(),
8041                workspace_id: None,
8042                root_id: None,
8043                provider: Some(selected.provider.clone()),
8044                model: Some(selected.model.clone()),
8045                tool_allowlist: Vec::new(),
8046                developer_instructions: None,
8047                external_tools: Vec::new(),
8048                selection_mode: Some(ModelSelectionMode::auto(
8049                    "test-router:default",
8050                    "test-router",
8051                    "Auto",
8052                    selected.clone(),
8053                    None,
8054                    None,
8055                )),
8056                runner: None,
8057            })
8058            .await
8059            .unwrap()
8060            .thread_id;
8061        let mut rx = runtime.subscribe_events();
8062        let turn_id = runtime
8063            .start_turn(StartTurnRequest {
8064                thread_id,
8065                message: "use explicit selection".to_string(),
8066                images: Vec::new(),
8067                provider_override: Some(selected.provider.clone()),
8068                model_override: Some(selected.model.clone()),
8069                reasoning_override: None,
8070                workspace: test_workspace(),
8071                instructions: InstructionBundle::default(),
8072                developer_context: None,
8073                task_ledger_required: false,
8074            })
8075            .await
8076            .unwrap();
8077
8078        let mut saw_routing_event = false;
8079        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8080            loop {
8081                let envelope = rx.recv().await.unwrap();
8082                if envelope.turn_id.as_deref() != Some(&turn_id) {
8083                    continue;
8084                }
8085                match envelope.event {
8086                    RoderEvent::InferenceRoutingDecision(_) => {
8087                        saw_routing_event = true;
8088                    }
8089                    RoderEvent::TurnCompleted(_) => break,
8090                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8091                    _ => {}
8092                }
8093            }
8094        })
8095        .await
8096        .unwrap();
8097
8098        assert!(!saw_routing_event);
8099        assert!(contexts.lock().unwrap().is_empty());
8100        let requests = requests.lock().unwrap();
8101        assert_eq!(requests.len(), 1);
8102        assert_eq!(requests[0].model, selected);
8103        let _ = std::fs::remove_dir_all(thread_root);
8104    }
8105
8106    #[tokio::test]
8107    async fn inference_router_is_bypassed_for_manual_selection_mode() {
8108        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8109        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8110        let selected = ModelSelection {
8111            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8112            model: "mock".to_string(),
8113        };
8114
8115        let mut builder = ExtensionRegistryBuilder::new();
8116        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8117            id: roder_api::catalog::PROVIDER_MOCK,
8118            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8119            requests: requests.clone(),
8120        }));
8121        builder.inference_router(Arc::new(StaticRouter {
8122            id: "test-router",
8123            decision: InferenceRoutingDecision::selected(
8124                "test-router",
8125                ModelSelection {
8126                    provider: "missing".to_string(),
8127                    model: "missing".to_string(),
8128                },
8129                "would route if called",
8130            ),
8131            contexts: contexts.clone(),
8132        }));
8133        let thread_root =
8134            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8135        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8136            base_path: thread_root.clone(),
8137        }));
8138        let runtime = Arc::new(
8139            Runtime::new(
8140                builder.build().unwrap(),
8141                RuntimeConfig {
8142                    default_provider: selected.provider.clone(),
8143                    default_model: selected.model.clone(),
8144                    inference_router: RuntimeInferenceRouterConfig {
8145                        enabled: true,
8146                        router_id: Some("test-router".to_string()),
8147                    },
8148                    ..RuntimeConfig::default()
8149                },
8150            )
8151            .unwrap(),
8152        );
8153        let thread_id = runtime
8154            .create_thread_with(CreateThreadRequest {
8155                title: Some("Routing manual".to_string()),
8156                workspace: test_workspace(),
8157                workspace_id: None,
8158                root_id: None,
8159                provider: Some(selected.provider.clone()),
8160                model: Some(selected.model.clone()),
8161                tool_allowlist: Vec::new(),
8162                developer_instructions: None,
8163                external_tools: Vec::new(),
8164                selection_mode: Some(ModelSelectionMode::manual(
8165                    selected.provider.clone(),
8166                    selected.model.clone(),
8167                    None,
8168                )),
8169                runner: None,
8170            })
8171            .await
8172            .unwrap()
8173            .thread_id;
8174        let mut rx = runtime.subscribe_events();
8175        let turn_id = runtime
8176            .start_turn(StartTurnRequest {
8177                thread_id,
8178                message: "use selected manual model".to_string(),
8179                images: Vec::new(),
8180                provider_override: None,
8181                model_override: None,
8182                reasoning_override: None,
8183                workspace: test_workspace(),
8184                instructions: InstructionBundle::default(),
8185                developer_context: None,
8186                task_ledger_required: false,
8187            })
8188            .await
8189            .unwrap();
8190
8191        let mut saw_routing_event = false;
8192        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8193            loop {
8194                let envelope = rx.recv().await.unwrap();
8195                if envelope.turn_id.as_deref() != Some(&turn_id) {
8196                    continue;
8197                }
8198                match envelope.event {
8199                    RoderEvent::InferenceRoutingDecision(_) => {
8200                        saw_routing_event = true;
8201                    }
8202                    RoderEvent::TurnCompleted(_) => break,
8203                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8204                    _ => {}
8205                }
8206            }
8207        })
8208        .await
8209        .unwrap();
8210
8211        assert!(!saw_routing_event);
8212        assert!(contexts.lock().unwrap().is_empty());
8213        let requests = requests.lock().unwrap();
8214        assert_eq!(requests.len(), 1);
8215        assert_eq!(requests[0].model, selected);
8216        let _ = std::fs::remove_dir_all(thread_root);
8217    }
8218
8219    #[test]
8220    fn enabled_inference_router_requires_registered_router() {
8221        let mut builder = ExtensionRegistryBuilder::new();
8222        builder.inference_engine(Arc::new(FakeInferenceEngine));
8223
8224        let err = match Runtime::new(
8225            builder.build().unwrap(),
8226            RuntimeConfig {
8227                inference_router: RuntimeInferenceRouterConfig {
8228                    enabled: true,
8229                    router_id: Some("missing-router".to_string()),
8230                },
8231                ..RuntimeConfig::default()
8232            },
8233        ) {
8234            Ok(_) => panic!("runtime should reject unknown inference router"),
8235            Err(err) => err,
8236        };
8237
8238        assert!(
8239            err.to_string()
8240                .contains("inference router \"missing-router\" is not registered")
8241        );
8242    }
8243
8244    #[tokio::test]
8245    async fn model_profile_routes_request_knobs_to_next_inference() {
8246        let request = captured_profile_request(RuntimeConfig {
8247            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8248            default_model: "gpt-5.5".to_string(),
8249            model_profiles: std::collections::HashMap::from([(
8250                "gpt-5.5".to_string(),
8251                test_model_profile("gpt-5.5"),
8252            )]),
8253            ..RuntimeConfig::default()
8254        })
8255        .await;
8256
8257        let tool_names = request
8258            .tools
8259            .iter()
8260            .map(|tool| tool.name.as_str())
8261            .collect::<Vec<_>>();
8262        assert!(tool_names.contains(&"apply_patch"));
8263        assert!(tool_names.contains(&"edit"));
8264        assert!(tool_names.contains(&"multi_edit"));
8265        assert!(tool_names.contains(&"write_file"));
8266        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8267        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8268        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8269        assert!(
8270            request
8271                .instructions
8272                .developer
8273                .as_deref()
8274                .unwrap_or_default()
8275                .contains("Use the provided context as the current working set")
8276        );
8277        assert_eq!(
8278            request
8279                .metadata
8280                .pointer("/modelProfile/schemaPolicy")
8281                .and_then(serde_json::Value::as_str),
8282            Some("standard_required_first")
8283        );
8284    }
8285
8286    #[tokio::test]
8287    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8288        let request = captured_profile_request(RuntimeConfig {
8289            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8290            default_model: "gpt-5.6-sol".to_string(),
8291            reasoning: Some(REASONING_ULTRA.to_string()),
8292            ..RuntimeConfig::default()
8293        })
8294        .await;
8295
8296        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8297        let developer = request
8298            .instructions
8299            .developer
8300            .as_deref()
8301            .expect("ultra developer instructions");
8302        assert!(developer.contains("Proactive multi-agent delegation is active"));
8303    }
8304
8305    #[tokio::test]
8306    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8307        let request = captured_profile_request(RuntimeConfig {
8308            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8309            default_model: "gpt-5.6-sol".to_string(),
8310            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8311            ..RuntimeConfig::default()
8312        })
8313        .await;
8314
8315        let developer = request
8316            .instructions
8317            .developer
8318            .as_deref()
8319            .expect("sol developer instructions");
8320        assert!(developer.contains("Do not spawn sub-agents unless"));
8321        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8322    }
8323
8324    #[tokio::test]
8325    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8326        let request = captured_profile_request(RuntimeConfig {
8327            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8328            default_model: "gpt-5.6-luna".to_string(),
8329            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8330            ..RuntimeConfig::default()
8331        })
8332        .await;
8333
8334        let developer = request
8335            .instructions
8336            .developer
8337            .as_deref()
8338            .unwrap_or_default();
8339        assert!(!developer.contains("Do not spawn sub-agents unless"));
8340        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8341    }
8342
8343    #[tokio::test]
8344    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8345        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8346        let request = captured_profile_request(RuntimeConfig {
8347            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8348            default_model: "gpt-5.6-luna".to_string(),
8349            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8350            ultra_mode: true,
8351            ..RuntimeConfig::default()
8352        })
8353        .await;
8354
8355        let developer = request
8356            .instructions
8357            .developer
8358            .as_deref()
8359            .expect("ultra mode developer instructions");
8360        assert!(developer.contains("Proactive multi-agent delegation is active"));
8361        assert!(developer.contains("spawn_agent"));
8362    }
8363
8364    #[tokio::test]
8365    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8366        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8367        // it to proactive without requiring Ultra reasoning effort.
8368        let request = captured_profile_request(RuntimeConfig {
8369            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8370            default_model: "gpt-5.6-sol".to_string(),
8371            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8372            ultra_mode: true,
8373            ..RuntimeConfig::default()
8374        })
8375        .await;
8376
8377        let developer = request
8378            .instructions
8379            .developer
8380            .as_deref()
8381            .expect("ultra mode sol developer instructions");
8382        assert!(developer.contains("Proactive multi-agent delegation is active"));
8383        assert!(!developer.contains("Do not spawn sub-agents unless"));
8384    }
8385
8386    #[tokio::test]
8387    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8388        let captured = Arc::new(StdMutex::new(None));
8389        let mut builder = ExtensionRegistryBuilder::new();
8390        builder.inference_engine(Arc::new(CapturingEngine {
8391            request: captured.clone(),
8392        }));
8393        let runtime = Arc::new(
8394            Runtime::new(
8395                builder.build().unwrap(),
8396                RuntimeConfig {
8397                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8398                    default_model: "gpt-5.5".to_string(),
8399                    ..RuntimeConfig::default()
8400                },
8401            )
8402            .unwrap(),
8403        );
8404
8405        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8406            let mut rx = runtime.subscribe_events();
8407            let turn_id = runtime
8408                .start_turn(StartTurnRequest {
8409                    thread_id: "thread-turn-context".to_string(),
8410                    message: "hello".to_string(),
8411                    images: Vec::new(),
8412                    provider_override: None,
8413                    model_override: None,
8414                    reasoning_override: None,
8415                    workspace: test_workspace(),
8416                    instructions: InstructionBundle::default(),
8417                    developer_context,
8418                    task_ledger_required: false,
8419                })
8420                .await
8421                .unwrap();
8422            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8423                loop {
8424                    let envelope = rx.recv().await.unwrap();
8425                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8426                        continue;
8427                    }
8428                    match envelope.event {
8429                        RoderEvent::TurnCompleted(_) => break,
8430                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8431                        _ => {}
8432                    }
8433                }
8434            })
8435            .await
8436            .unwrap();
8437        }
8438
8439        run_turn(
8440            &runtime,
8441            Some("Connected accounts: example-service.".to_string()),
8442        )
8443        .await;
8444        let request = captured.lock().unwrap().clone().unwrap();
8445        assert_eq!(
8446            request.instructions.developer_context.as_deref(),
8447            Some("Connected accounts: example-service.")
8448        );
8449
8450        // The context is per-turn only: the next turn on the same thread
8451        // without a developerContext must not see the previous one.
8452        run_turn(&runtime, None).await;
8453        let request = captured.lock().unwrap().clone().unwrap();
8454        assert_eq!(request.instructions.developer_context, None);
8455    }
8456
8457    #[tokio::test]
8458    async fn tool_search_overrides_route_to_next_inference_request() {
8459        let request = captured_profile_request(RuntimeConfig {
8460            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8461            default_model: "gpt-5.4".to_string(),
8462            tool_search: ToolSearchConfig {
8463                mode: roder_api::inference::ToolSearchMode::Auto,
8464                max_catalog_items: Some(100),
8465                ..ToolSearchConfig::default()
8466            },
8467            provider_tool_search: std::collections::HashMap::from([(
8468                roder_api::catalog::PROVIDER_MOCK.to_string(),
8469                roder_api::inference::ToolSearchConfigOverlay {
8470                    include_skills: Some(false),
8471                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8472                    ..Default::default()
8473                },
8474            )]),
8475            model_tool_search: std::collections::HashMap::from([(
8476                "gpt-5.4".to_string(),
8477                roder_api::inference::ToolSearchConfigOverlay {
8478                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8479                    max_catalog_items: Some(25),
8480                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8481                    ..Default::default()
8482                },
8483            )]),
8484            ..RuntimeConfig::default()
8485        })
8486        .await;
8487
8488        assert_eq!(
8489            request.runtime.tool_search.mode,
8490            roder_api::inference::ToolSearchMode::ProviderNative
8491        );
8492        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8493        assert_eq!(
8494            request.runtime.tool_search.provider_variant,
8495            roder_api::inference::ToolSearchProviderVariant::Bm25
8496        );
8497    }
8498
8499    #[tokio::test]
8500    async fn context_entrypoint_hints_use_turn_workspace() {
8501        let process_workspace = runtime_test_workspace("entrypoint-process");
8502        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8503        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8504        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8505        std::fs::write(
8506            process_workspace.join("src/sidebar-thread-groups.ts"),
8507            "export const desktopLeak = true;\n",
8508        )
8509        .unwrap();
8510        std::fs::write(
8511            thread_workspace.join("src/voice-plan-feedback.ts"),
8512            "export const voicePlanFeedback = true;\n",
8513        )
8514        .unwrap();
8515
8516        let captured = Arc::new(StdMutex::new(None));
8517        let mut builder = ExtensionRegistryBuilder::new();
8518        builder.inference_engine(Arc::new(CapturingEngine {
8519            request: captured.clone(),
8520        }));
8521        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8522            process_workspace.clone(),
8523        )));
8524        let runtime = Arc::new(
8525            Runtime::new(
8526                builder.build().unwrap(),
8527                RuntimeConfig {
8528                    workspace: Some(process_workspace.display().to_string()),
8529                    ..RuntimeConfig::default()
8530                },
8531            )
8532            .unwrap(),
8533        );
8534        let mut rx = runtime.subscribe_events();
8535
8536        let turn_id = runtime
8537            .start_turn(StartTurnRequest {
8538                thread_id: "thread-workspace-entrypoint".to_string(),
8539                message: "investigate voice plan feedback".to_string(),
8540                images: Vec::new(),
8541                provider_override: None,
8542                model_override: None,
8543                reasoning_override: None,
8544                workspace: thread_workspace.display().to_string(),
8545                instructions: crate::instructions::default_instructions(),
8546                developer_context: None,
8547                task_ledger_required: false,
8548            })
8549            .await
8550            .unwrap();
8551
8552        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8553            loop {
8554                let envelope = rx.recv().await.unwrap();
8555                if envelope.turn_id.as_deref() != Some(&turn_id) {
8556                    continue;
8557                }
8558                match envelope.event {
8559                    RoderEvent::TurnCompleted(_) => break,
8560                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8561                    _ => {}
8562                }
8563            }
8564        })
8565        .await
8566        .unwrap();
8567
8568        let request = captured.lock().unwrap().clone().expect("captured request");
8569        let transcript_text = request
8570            .transcript
8571            .iter()
8572            .map(|item| match item {
8573                TranscriptItem::UserMessage(message) => message.text.as_str(),
8574                _ => "",
8575            })
8576            .collect::<Vec<_>>()
8577            .join("\n");
8578        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8579        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8580
8581        let _ = std::fs::remove_dir_all(process_workspace);
8582        let _ = std::fs::remove_dir_all(thread_workspace);
8583    }
8584
8585    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8586        let path =
8587            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8588        let _ = std::fs::remove_dir_all(&path);
8589        std::fs::create_dir_all(&path).unwrap();
8590        path
8591    }
8592
8593    #[tokio::test]
8594    async fn model_profile_user_model_knobs_override_profile_defaults() {
8595        let request = captured_profile_request(RuntimeConfig {
8596            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8597            default_model: "gpt-5.5".to_string(),
8598            reasoning: Some(REASONING_HIGH.to_string()),
8599            auto_compact_token_limit: Some(999),
8600            model_edit_tools: std::collections::HashMap::from([(
8601                "gpt-5.5".to_string(),
8602                EDIT_TOOL_PATCH.to_string(),
8603            )]),
8604            model_parallel_tool_calls: std::collections::HashMap::from([(
8605                "gpt-5.5".to_string(),
8606                true,
8607            )]),
8608            model_profiles: std::collections::HashMap::from([(
8609                "gpt-5.5".to_string(),
8610                test_model_profile("gpt-5.5"),
8611            )]),
8612            ..RuntimeConfig::default()
8613        })
8614        .await;
8615
8616        let tool_names = request
8617            .tools
8618            .iter()
8619            .map(|tool| tool.name.as_str())
8620            .collect::<Vec<_>>();
8621        assert!(tool_names.contains(&"apply_patch"));
8622        assert!(!tool_names.contains(&"edit"));
8623        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8624        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8625        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8626    }
8627
8628    #[tokio::test]
8629    async fn runtime_tool_allowlist_filters_advertised_tools() {
8630        let request = captured_profile_request(RuntimeConfig {
8631            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8632            default_model: "gpt-5.5".to_string(),
8633            tool_allowlist: vec!["edit".to_string()],
8634            model_profiles: std::collections::HashMap::from([(
8635                "gpt-5.5".to_string(),
8636                test_model_profile("gpt-5.5"),
8637            )]),
8638            ..RuntimeConfig::default()
8639        })
8640        .await;
8641
8642        let tool_names = request
8643            .tools
8644            .iter()
8645            .map(|tool| tool.name.as_str())
8646            .collect::<Vec<_>>();
8647        assert_eq!(tool_names, vec!["edit"]);
8648    }
8649
8650    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8651    /// overrides and returns the captured inference request.
8652    async fn captured_thread_override_request(
8653        runtime: &Arc<Runtime>,
8654        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8655        tool_allowlist: Vec<String>,
8656        developer_instructions: Option<String>,
8657        external_tools: Vec<ToolSpec>,
8658    ) -> AgentInferenceRequest {
8659        let thread_id = runtime
8660            .create_thread_with(CreateThreadRequest {
8661                title: Some("Thread overrides".to_string()),
8662                workspace: test_workspace(),
8663                workspace_id: None,
8664                root_id: None,
8665                provider: None,
8666                model: None,
8667                selection_mode: None,
8668                tool_allowlist,
8669                developer_instructions,
8670                external_tools,
8671                runner: None,
8672            })
8673            .await
8674            .unwrap()
8675            .thread_id;
8676        let mut rx = runtime.subscribe_events();
8677        let turn_id = runtime
8678            .start_turn(StartTurnRequest {
8679                thread_id,
8680                message: "hello".to_string(),
8681                images: Vec::new(),
8682                provider_override: None,
8683                model_override: None,
8684                reasoning_override: None,
8685                workspace: test_workspace(),
8686                instructions: crate::default_instructions(),
8687                developer_context: None,
8688                task_ledger_required: false,
8689            })
8690            .await
8691            .unwrap();
8692        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8693            loop {
8694                let envelope = rx.recv().await.unwrap();
8695                if envelope.turn_id.as_deref() != Some(&turn_id) {
8696                    continue;
8697                }
8698                match envelope.event {
8699                    RoderEvent::TurnCompleted(_) => break,
8700                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8701                    _ => {}
8702                }
8703            }
8704        })
8705        .await
8706        .unwrap();
8707        requests.lock().unwrap().pop().expect("captured request")
8708    }
8709
8710    #[tokio::test]
8711    async fn thread_tool_allowlist_filters_only_that_thread() {
8712        let requests = Arc::new(StdMutex::new(Vec::new()));
8713        let thread_root =
8714            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8715        let mut builder = ExtensionRegistryBuilder::new();
8716        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8717            requests: requests.clone(),
8718        }));
8719        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8720            base_path: thread_root.clone(),
8721        }));
8722        builder.tool_contributor(Arc::new(ProfileToolContributor));
8723        let runtime = Arc::new(
8724            Runtime::new(
8725                builder.build().unwrap(),
8726                RuntimeConfig {
8727                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8728                    default_model: "gpt-5.5".to_string(),
8729                    model_profiles: std::collections::HashMap::from([(
8730                        "gpt-5.5".to_string(),
8731                        test_model_profile("gpt-5.5"),
8732                    )]),
8733                    ..RuntimeConfig::default()
8734                },
8735            )
8736            .unwrap(),
8737        );
8738
8739        let allowlisted = captured_thread_override_request(
8740            &runtime,
8741            &requests,
8742            vec!["edit".to_string()],
8743            None,
8744            Vec::new(),
8745        )
8746        .await;
8747        let unrestricted =
8748            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8749                .await;
8750
8751        let allowlisted_names = allowlisted
8752            .tools
8753            .iter()
8754            .map(|tool| tool.name.as_str())
8755            .collect::<Vec<_>>();
8756        assert_eq!(allowlisted_names, vec!["edit"]);
8757        let unrestricted_names = unrestricted
8758            .tools
8759            .iter()
8760            .map(|tool| tool.name.as_str())
8761            .collect::<Vec<_>>();
8762        assert!(unrestricted_names.contains(&"edit"));
8763        assert!(unrestricted_names.len() > 1);
8764
8765        let _ = std::fs::remove_dir_all(thread_root);
8766    }
8767
8768    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8769    fn runtime_with_edit_allowlist(
8770        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8771        thread_root: &std::path::Path,
8772    ) -> Arc<Runtime> {
8773        let mut builder = ExtensionRegistryBuilder::new();
8774        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8775            requests: requests.clone(),
8776        }));
8777        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8778            base_path: thread_root.to_path_buf(),
8779        }));
8780        builder.tool_contributor(Arc::new(ProfileToolContributor));
8781        Arc::new(
8782            Runtime::new(
8783                builder.build().unwrap(),
8784                RuntimeConfig {
8785                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8786                    default_model: "gpt-5.5".to_string(),
8787                    tool_allowlist: vec!["edit".to_string()],
8788                    model_profiles: std::collections::HashMap::from([(
8789                        "gpt-5.5".to_string(),
8790                        test_model_profile("gpt-5.5"),
8791                    )]),
8792                    ..RuntimeConfig::default()
8793                },
8794            )
8795            .unwrap(),
8796        )
8797    }
8798
8799    #[tokio::test]
8800    async fn runtime_and_thread_allowlists_intersect() {
8801        let requests = Arc::new(StdMutex::new(Vec::new()));
8802        let thread_root = std::env::temp_dir().join(format!(
8803            "roder-allowlist-intersect-{}",
8804            uuid::Uuid::new_v4()
8805        ));
8806        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8807
8808        let request = captured_thread_override_request(
8809            &runtime,
8810            &requests,
8811            vec!["edit".to_string(), "write_file".to_string()],
8812            None,
8813            Vec::new(),
8814        )
8815        .await;
8816
8817        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8818        let names = request
8819            .tools
8820            .iter()
8821            .map(|tool| tool.name.as_str())
8822            .collect::<Vec<_>>();
8823        assert_eq!(names, vec!["edit"]);
8824
8825        let _ = std::fs::remove_dir_all(thread_root);
8826    }
8827
8828    #[tokio::test]
8829    async fn route_tool_call_denies_tools_outside_allowlists() {
8830        let requests = Arc::new(StdMutex::new(Vec::new()));
8831        let thread_root =
8832            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8833        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8834        let thread_id = runtime
8835            .create_thread_with(CreateThreadRequest {
8836                title: Some("Dispatch allowlist".to_string()),
8837                workspace: test_workspace(),
8838                workspace_id: None,
8839                root_id: None,
8840                provider: None,
8841                model: None,
8842                selection_mode: None,
8843                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8844                developer_instructions: None,
8845                external_tools: Vec::new(),
8846                runner: None,
8847            })
8848            .await
8849            .unwrap()
8850            .thread_id;
8851
8852        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8853        let result = runtime
8854            .route_tool_call(
8855                &thread_id,
8856                &"turn-allowlist-dispatch".to_string(),
8857                roder_api::inference::ToolCallCompleted {
8858                    id: "call-1".to_string(),
8859                    name: "write_file".to_string(),
8860                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8861                },
8862                None,
8863                None,
8864            )
8865            .await
8866            .unwrap();
8867
8868        assert!(result.is_error);
8869        assert!(
8870            result
8871                .result
8872                .contains("not permitted by the tool allowlist"),
8873            "unexpected result: {}",
8874            result.result
8875        );
8876
8877        let _ = std::fs::remove_dir_all(thread_root);
8878    }
8879
8880    #[tokio::test]
8881    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8882        let requests = Arc::new(StdMutex::new(Vec::new()));
8883        let thread_root =
8884            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8885        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8886
8887        // A response that mixes agent_swarm with another tool is denied wholesale:
8888        // both calls get an error result with retry guidance, and no tool runs.
8889        let results = runtime
8890            .route_tool_calls(
8891                &"thread-swarm".to_string(),
8892                &"turn-swarm".to_string(),
8893                vec![
8894                    roder_api::inference::ToolCallCompleted {
8895                        id: "swarm-1".to_string(),
8896                        name: "agent_swarm".to_string(),
8897                        arguments: "{}".to_string(),
8898                    },
8899                    roder_api::inference::ToolCallCompleted {
8900                        id: "read-1".to_string(),
8901                        name: "read_file".to_string(),
8902                        arguments: "{}".to_string(),
8903                    },
8904                ],
8905                true,
8906                None,
8907                None,
8908            )
8909            .await
8910            .unwrap();
8911
8912        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8913        assert_eq!(results[0].id, "swarm-1");
8914        assert_eq!(results[1].id, "read-1");
8915        for result in &results {
8916            assert!(result.is_error);
8917            assert!(
8918                result.result.contains("only tool call"),
8919                "unexpected result: {}",
8920                result.result
8921            );
8922        }
8923
8924        let _ = std::fs::remove_dir_all(thread_root);
8925    }
8926
8927    #[tokio::test]
8928    async fn route_tool_calls_emits_agent_swarm_started_event() {
8929        let requests = Arc::new(StdMutex::new(Vec::new()));
8930        let thread_root =
8931            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8932        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8933        let mut events = runtime.subscribe_events();
8934
8935        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8936        // the event bus before dispatch, with the child count parsed from args.
8937        let _ = runtime
8938            .route_tool_calls(
8939                &"thread-swarm".to_string(),
8940                &"turn-swarm".to_string(),
8941                vec![roder_api::inference::ToolCallCompleted {
8942                    id: "swarm-1".to_string(),
8943                    name: "agent_swarm".to_string(),
8944                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8945                        .to_string(),
8946                }],
8947                true,
8948                None,
8949                None,
8950            )
8951            .await
8952            .unwrap();
8953
8954        let mut started_child_count = None;
8955        for _ in 0..16 {
8956            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8957                .await
8958                .unwrap()
8959                .unwrap();
8960            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8961                started_child_count = Some(event.child_count);
8962                assert_eq!(event.tool_id, "swarm-1");
8963                break;
8964            }
8965        }
8966        assert_eq!(started_child_count, Some(2));
8967
8968        let _ = std::fs::remove_dir_all(thread_root);
8969    }
8970
8971    #[test]
8972    fn agent_swarm_child_count_sums_items_and_resumes() {
8973        assert_eq!(
8974            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8975            3
8976        );
8977        assert_eq!(
8978            agent_swarm_child_count(
8979                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8980            ),
8981            2
8982        );
8983        // Malformed input is lenient.
8984        assert_eq!(agent_swarm_child_count("not json"), 0);
8985    }
8986
8987    #[test]
8988    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8989        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8990        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8991        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8992        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8993        // Not a swarm result.
8994        assert_eq!(parse_swarm_counts("just text"), None);
8995    }
8996
8997    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8998    struct SignalledFailureEngine {
8999        started: tokio::sync::mpsc::UnboundedSender<()>,
9000        proceed: Arc<tokio::sync::Notify>,
9001    }
9002
9003    #[async_trait::async_trait]
9004    impl InferenceEngine for SignalledFailureEngine {
9005        fn id(&self) -> String {
9006            roder_api::catalog::PROVIDER_MOCK.to_string()
9007        }
9008
9009        fn capabilities(&self) -> InferenceCapabilities {
9010            InferenceCapabilities::coding_agent_default()
9011        }
9012
9013        async fn list_models(
9014            &self,
9015            _ctx: InferenceProviderContext<'_>,
9016        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9017            Ok(roder_api::catalog::models_for_provider(
9018                roder_api::catalog::PROVIDER_MOCK,
9019                true,
9020            ))
9021        }
9022
9023        async fn stream_turn(
9024            &self,
9025            _ctx: InferenceTurnContext<'_>,
9026            _request: AgentInferenceRequest,
9027        ) -> anyhow::Result<InferenceEventStream> {
9028            let _ = self.started.send(());
9029            self.proceed.notified().await;
9030            anyhow::bail!("engine failed mid-turn")
9031        }
9032    }
9033
9034    #[tokio::test]
9035    async fn failed_turn_sweeps_pending_external_tool_calls() {
9036        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9037        let proceed = Arc::new(tokio::sync::Notify::new());
9038        let mut builder = ExtensionRegistryBuilder::new();
9039        builder.inference_engine(Arc::new(SignalledFailureEngine {
9040            started: started_tx,
9041            proceed: proceed.clone(),
9042        }));
9043        let runtime =
9044            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9045        let mut rx = runtime.subscribe_events();
9046        let turn_id = runtime
9047            .start_turn(StartTurnRequest {
9048                thread_id: "thread-sweep".to_string(),
9049                message: "go".to_string(),
9050                images: Vec::new(),
9051                provider_override: None,
9052                model_override: None,
9053                reasoning_override: None,
9054                workspace: test_workspace(),
9055                instructions: InstructionBundle {
9056                    system: None,
9057                    developer: None,
9058                    developer_context: None,
9059                },
9060                developer_context: None,
9061                task_ledger_required: false,
9062            })
9063            .await
9064            .unwrap();
9065        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9066            .await
9067            .unwrap()
9068            .unwrap();
9069
9070        let (tx, _pending_rx) = oneshot::channel();
9071        runtime.pending_external_tool_calls.lock().await.insert(
9072            "exttool-sweep-test".to_string(),
9073            PendingExternalToolCall {
9074                thread_id: "thread-sweep".to_string(),
9075                turn_id: turn_id.clone(),
9076                tool_id: "call-1".to_string(),
9077                tool_name: "acme_lookup".to_string(),
9078                tx,
9079            },
9080        );
9081        proceed.notify_one();
9082
9083        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9084            loop {
9085                let envelope = rx.recv().await.unwrap();
9086                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9087                    && event.request_id == "exttool-sweep-test"
9088                {
9089                    break event.outcome;
9090                }
9091            }
9092        })
9093        .await
9094        .expect("turn failure must resolve pending external tool calls");
9095        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9096        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9097    }
9098
9099    #[tokio::test]
9100    async fn thread_developer_instructions_layer_under_harness_prompt() {
9101        let requests = Arc::new(StdMutex::new(Vec::new()));
9102        let thread_root = std::env::temp_dir().join(format!(
9103            "roder-thread-instructions-{}",
9104            uuid::Uuid::new_v4()
9105        ));
9106        let mut builder = ExtensionRegistryBuilder::new();
9107        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9108            requests: requests.clone(),
9109        }));
9110        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9111            base_path: thread_root.clone(),
9112        }));
9113        builder.tool_contributor(Arc::new(ProfileToolContributor));
9114        let runtime =
9115            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9116
9117        let request = captured_thread_override_request(
9118            &runtime,
9119            &requests,
9120            Vec::new(),
9121            Some("You are embedded in a host app.".to_string()),
9122            Vec::new(),
9123        )
9124        .await;
9125
9126        let system = request.instructions.system.expect("system instructions");
9127        assert!(system.starts_with("You are Roder"));
9128        let developer = request
9129            .instructions
9130            .developer
9131            .expect("developer instructions");
9132        assert!(developer.starts_with("You are embedded in a host app."));
9133
9134        let plain =
9135            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9136                .await;
9137        assert_eq!(plain.instructions.developer, None);
9138
9139        let _ = std::fs::remove_dir_all(thread_root);
9140    }
9141
9142    #[tokio::test]
9143    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9144        let requests = Arc::new(StdMutex::new(Vec::new()));
9145        let thread_root = std::env::temp_dir().join(format!(
9146            "roder-thread-external-tools-{}",
9147            uuid::Uuid::new_v4()
9148        ));
9149        let mut builder = ExtensionRegistryBuilder::new();
9150        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9151            requests: requests.clone(),
9152        }));
9153        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9154            base_path: thread_root.clone(),
9155        }));
9156        builder.tool_contributor(Arc::new(ProfileToolContributor));
9157        let runtime =
9158            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9159
9160        let external_tools = vec![
9161            ToolSpec {
9162                name: "acme_lookup".to_string(),
9163                description: "Look up Acme workspace state.".to_string(),
9164                parameters: serde_json::json!({
9165                    "type": "object",
9166                    "properties": { "query": { "type": "string" } },
9167                    "required": ["query"]
9168                }),
9169            },
9170            ToolSpec {
9171                name: "edit".to_string(),
9172                description: "Host-managed edit.".to_string(),
9173                parameters: serde_json::json!({ "type": "object" }),
9174            },
9175        ];
9176        let request =
9177            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9178                .await;
9179
9180        let acme = request
9181            .tools
9182            .iter()
9183            .find(|tool| tool.name == "acme_lookup")
9184            .expect("external tool advertised");
9185        assert_eq!(acme.description, "Look up Acme workspace state.");
9186        assert_eq!(acme.parameters["required"][0], "query");
9187        let edits = request
9188            .tools
9189            .iter()
9190            .filter(|tool| tool.name == "edit")
9191            .collect::<Vec<_>>();
9192        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9193        assert_eq!(edits[0].description, "Host-managed edit.");
9194
9195        let plain =
9196            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9197                .await;
9198        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9199        let plain_edit = plain
9200            .tools
9201            .iter()
9202            .find(|tool| tool.name == "edit")
9203            .expect("builtin edit advertised on plain thread");
9204        assert_eq!(plain_edit.description, "edit test tool");
9205
9206        let _ = std::fs::remove_dir_all(thread_root);
9207    }
9208
9209    #[tokio::test]
9210    async fn model_switch_injects_summary_and_records_profile_segments() {
9211        let requests = Arc::new(StdMutex::new(Vec::new()));
9212        let thread_root = std::env::temp_dir().join(format!(
9213            "roder-model-switch-thread-{}",
9214            uuid::Uuid::new_v4()
9215        ));
9216        let mut builder = ExtensionRegistryBuilder::new();
9217        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9218            requests: requests.clone(),
9219        }));
9220        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9221            base_path: thread_root.clone(),
9222        }));
9223        builder.tool_contributor(Arc::new(ProfileToolContributor));
9224        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9225        claude_profile.provider_family = ProviderFamily::Anthropic;
9226        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9227        let runtime = Arc::new(
9228            Runtime::new(
9229                builder.build().unwrap(),
9230                RuntimeConfig {
9231                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9232                    default_model: "gpt-5.5".to_string(),
9233                    model_profiles: std::collections::HashMap::from([
9234                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9235                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9236                    ]),
9237                    ..RuntimeConfig::default()
9238                },
9239            )
9240            .unwrap(),
9241        );
9242        let thread_id = runtime
9243            .create_thread_with(CreateThreadRequest {
9244                title: Some("Model switch".to_string()),
9245                workspace: test_workspace(),
9246                workspace_id: None,
9247                root_id: None,
9248                provider: None,
9249                model: None,
9250                selection_mode: None,
9251                tool_allowlist: Vec::new(),
9252                developer_instructions: None,
9253                external_tools: Vec::new(),
9254                runner: None,
9255            })
9256            .await
9257            .unwrap()
9258            .thread_id;
9259        let mut rx = runtime.subscribe_events();
9260        for (message, model_override) in [
9261            ("first turn", None),
9262            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9263        ] {
9264            let turn_id = runtime
9265                .start_turn(StartTurnRequest {
9266                    thread_id: thread_id.clone(),
9267                    message: message.to_string(),
9268                    images: Vec::new(),
9269                    provider_override: None,
9270                    model_override,
9271                    reasoning_override: None,
9272                    workspace: test_workspace(),
9273                    instructions: InstructionBundle::default(),
9274                    developer_context: None,
9275                    task_ledger_required: false,
9276                })
9277                .await
9278                .unwrap();
9279            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9280                loop {
9281                    let envelope = rx.recv().await.unwrap();
9282                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9283                        continue;
9284                    }
9285                    match envelope.event {
9286                        RoderEvent::TurnCompleted(_) => break,
9287                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9288                        _ => {}
9289                    }
9290                }
9291            })
9292            .await
9293            .unwrap();
9294        }
9295
9296        let captured = requests.lock().unwrap().clone();
9297        assert_eq!(captured.len(), 2);
9298        assert!(captured[1].transcript.iter().any(|item| {
9299            matches!(
9300                item,
9301                TranscriptItem::UserMessage(message)
9302                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9303                        && message.text.contains("previous profile mock/gpt-5.5")
9304                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9305                        && message.text.contains("Available tools now:")
9306            )
9307        }));
9308
9309        let snapshot = runtime
9310            .thread_store
9311            .as_ref()
9312            .unwrap()
9313            .load_thread(&thread_id)
9314            .await
9315            .unwrap()
9316            .unwrap();
9317        let trace_segments = snapshot
9318            .turns
9319            .iter()
9320            .flat_map(|turn| &turn.items)
9321            .filter(|item| {
9322                matches!(
9323                    item,
9324                    TranscriptItem::ProviderMetadata(value)
9325                        if value.get("kind").and_then(serde_json::Value::as_str)
9326                            == Some(MODEL_PROFILE_TRACE_KIND)
9327                            && value.get("segment").and_then(serde_json::Value::as_str)
9328                                == Some("assistant")
9329                )
9330            })
9331            .count();
9332        assert!(trace_segments >= 2);
9333        let _ = std::fs::remove_dir_all(thread_root);
9334    }
9335
9336    struct CountingTaskTool {
9337        calls: Arc<StdMutex<u32>>,
9338    }
9339
9340    #[async_trait::async_trait]
9341    impl ToolExecutor for CountingTaskTool {
9342        fn spec(&self) -> ToolSpec {
9343            ToolSpec {
9344                name: "task".to_string(),
9345                description: "Dispatch a test subagent.".to_string(),
9346                parameters: serde_json::json!({
9347                    "type": "object",
9348                    "properties": {
9349                        "description": { "type": "string" },
9350                        "prompt": { "type": "string" },
9351                        "parent_deadline_seconds": { "type": "integer" }
9352                    },
9353                    "required": ["description", "prompt"],
9354                    "additionalProperties": false
9355                }),
9356            }
9357        }
9358
9359        async fn execute(
9360            &self,
9361            _ctx: ToolExecutionContext,
9362            call: ToolCall,
9363        ) -> anyhow::Result<ToolResult> {
9364            *self.calls.lock().unwrap() += 1;
9365            Ok(ToolResult {
9366                id: call.id,
9367                name: call.name,
9368                text: "started child".to_string(),
9369                data: serde_json::json!({}),
9370                is_error: false,
9371            })
9372        }
9373    }
9374
9375    #[tokio::test]
9376    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9377        let captured = Arc::new(StdMutex::new(None));
9378        let mut builder = ExtensionRegistryBuilder::new();
9379        builder.inference_engine(Arc::new(CapturingEngine {
9380            request: captured.clone(),
9381        }));
9382        let runtime = Arc::new(
9383            Runtime::new(
9384                builder.build().unwrap(),
9385                RuntimeConfig {
9386                    runtime_profile: RuntimeProfile::NonInteractive,
9387                    ..RuntimeConfig::default()
9388                },
9389            )
9390            .unwrap(),
9391        );
9392        let mut rx = runtime.subscribe_events();
9393        let turn_id = runtime
9394            .start_turn(StartTurnRequest {
9395                thread_id: "thread-profile".to_string(),
9396                message: "work unattended".to_string(),
9397                images: Vec::new(),
9398                provider_override: None,
9399                model_override: None,
9400                reasoning_override: None,
9401                workspace: test_workspace(),
9402                instructions: InstructionBundle {
9403                    system: None,
9404                    developer: Some("base developer".to_string()),
9405                    developer_context: None,
9406                },
9407                developer_context: None,
9408                task_ledger_required: false,
9409            })
9410            .await
9411            .unwrap();
9412
9413        let mut observed_profile = None;
9414        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9415            loop {
9416                let envelope = rx.recv().await.unwrap();
9417                if envelope.turn_id.as_deref() != Some(&turn_id) {
9418                    continue;
9419                }
9420                match envelope.event {
9421                    RoderEvent::TurnStarted(event) => {
9422                        observed_profile = Some(event.runtime_profile);
9423                    }
9424                    RoderEvent::TurnCompleted(_) => break,
9425                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9426                    _ => {}
9427                }
9428            }
9429        })
9430        .await
9431        .unwrap();
9432
9433        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9434        let request = captured.lock().unwrap().clone().unwrap();
9435        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9436        let developer = request.instructions.developer.unwrap();
9437        assert!(developer.contains("base developer"));
9438        assert!(developer.contains("non-interactive profile"));
9439    }
9440
9441    #[tokio::test]
9442    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9443        let workspace = runtime_test_workspace("global-policy-mode");
9444        let thread_root = workspace.join("threads");
9445        let mut builder = ExtensionRegistryBuilder::new();
9446        builder.inference_engine(Arc::new(FakeInferenceEngine));
9447        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9448            base_path: thread_root.clone(),
9449        }));
9450        let runtime = Runtime::new(
9451            builder.build().unwrap(),
9452            RuntimeConfig {
9453                workspace: Some(workspace.display().to_string()),
9454                ..Default::default()
9455            },
9456        )
9457        .unwrap();
9458
9459        runtime
9460            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9461            .await
9462            .unwrap();
9463
9464        assert!(!thread_root.join("runtime").exists());
9465        let _ = std::fs::remove_dir_all(workspace);
9466    }
9467
9468    #[tokio::test]
9469    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9470        let captured = Arc::new(StdMutex::new(None));
9471        let mut builder = ExtensionRegistryBuilder::new();
9472        builder.inference_engine(Arc::new(CapturingEngine {
9473            request: captured.clone(),
9474        }));
9475        builder.tool_contributor(Arc::new(
9476            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9477        ));
9478        let runtime = Arc::new(
9479            Runtime::new(
9480                builder.build().unwrap(),
9481                RuntimeConfig {
9482                    runtime_profile: RuntimeProfile::Eval,
9483                    policy_mode: PolicyMode::Bypass,
9484                    agent_swarm_mode: false,
9485                    ultra_mode: false,
9486                    ..RuntimeConfig::default()
9487                },
9488            )
9489            .unwrap(),
9490        );
9491        let mut rx = runtime.subscribe_events();
9492        let turn_id = runtime
9493            .start_turn(StartTurnRequest {
9494                thread_id: "thread-ledger".to_string(),
9495                message: "decomposed work".to_string(),
9496                images: Vec::new(),
9497                provider_override: None,
9498                model_override: None,
9499                reasoning_override: None,
9500                workspace: test_workspace(),
9501                instructions: InstructionBundle::default(),
9502                developer_context: None,
9503                task_ledger_required: true,
9504            })
9505            .await
9506            .unwrap();
9507
9508        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9509            loop {
9510                let envelope = rx.recv().await.unwrap();
9511                if envelope.turn_id.as_deref() == Some(&turn_id)
9512                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9513                {
9514                    break;
9515                }
9516            }
9517        })
9518        .await
9519        .unwrap();
9520
9521        let request = captured.lock().unwrap().clone().unwrap();
9522        let developer = request.instructions.developer.unwrap();
9523        assert!(developer.contains("Task Ledger Required"));
9524        assert!(developer.contains("task_ledger.update"));
9525        let tool_names: Vec<_> = request
9526            .tools
9527            .iter()
9528            .map(|tool| tool.name.as_str())
9529            .collect();
9530        assert!(
9531            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9532            "tool names: {tool_names:?}"
9533        );
9534        assert_eq!(
9535            request.tool_choice,
9536            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9537        );
9538        assert_eq!(request.tools.len(), 1);
9539        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9540    }
9541
9542    #[tokio::test]
9543    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9544        let requests = Arc::new(StdMutex::new(Vec::new()));
9545        let mut builder = ExtensionRegistryBuilder::new();
9546        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9547            calls: StdMutex::new(0),
9548            requests: requests.clone(),
9549        }));
9550        builder.tool_contributor(Arc::new(
9551            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9552        ));
9553        let runtime = Arc::new(
9554            Runtime::new(
9555                builder.build().unwrap(),
9556                RuntimeConfig {
9557                    runtime_profile: RuntimeProfile::Eval,
9558                    policy_mode: PolicyMode::Bypass,
9559                    agent_swarm_mode: false,
9560                    ultra_mode: false,
9561                    ..RuntimeConfig::default()
9562                },
9563            )
9564            .unwrap(),
9565        );
9566        let mut rx = runtime.subscribe_events();
9567        let turn_id = runtime
9568            .start_turn(StartTurnRequest {
9569                thread_id: "thread-ledger-completion".to_string(),
9570                message: "write the answer file".to_string(),
9571                images: Vec::new(),
9572                provider_override: None,
9573                model_override: None,
9574                reasoning_override: None,
9575                workspace: test_workspace(),
9576                instructions: InstructionBundle::default(),
9577                developer_context: None,
9578                task_ledger_required: true,
9579            })
9580            .await
9581            .unwrap();
9582
9583        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9584            loop {
9585                let envelope = rx.recv().await.unwrap();
9586                if envelope.turn_id.as_deref() != Some(&turn_id) {
9587                    continue;
9588                }
9589                match envelope.event {
9590                    RoderEvent::TurnCompleted(_) => break,
9591                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9592                    _ => {}
9593                }
9594            }
9595        })
9596        .await
9597        .unwrap();
9598
9599        let requests = requests.lock().unwrap().clone();
9600        assert_eq!(requests.len(), 4);
9601        assert!(requests[2].transcript.iter().any(|item| {
9602            matches!(
9603                item,
9604                TranscriptItem::UserMessage(message)
9605                    if message.text.contains("Task Ledger Completion Required")
9606                        && message.text.contains("Write /app/result.txt")
9607            )
9608        }));
9609        assert!(requests[3].transcript.iter().any(|item| {
9610            matches!(
9611                item,
9612                TranscriptItem::ToolResult(result)
9613                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9614                        && result.result.contains("Task ledger: 2/2 completed")
9615            )
9616        }));
9617    }
9618
9619    #[tokio::test]
9620    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9621        let requests = Arc::new(StdMutex::new(Vec::new()));
9622        let mut builder = ExtensionRegistryBuilder::new();
9623        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9624            calls: StdMutex::new(0),
9625            requests: requests.clone(),
9626        }));
9627        builder.tool_contributor(Arc::new(
9628            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9629        ));
9630        let runtime = Arc::new(
9631            Runtime::new(
9632                builder.build().unwrap(),
9633                RuntimeConfig {
9634                    runtime_profile: RuntimeProfile::Eval,
9635                    policy_mode: PolicyMode::Bypass,
9636                    agent_swarm_mode: false,
9637                    ultra_mode: false,
9638                    turn_deadline_seconds: Some(120),
9639                    ..RuntimeConfig::default()
9640                },
9641            )
9642            .unwrap(),
9643        );
9644        let mut rx = runtime.subscribe_events();
9645        let turn_id = runtime
9646            .start_turn(StartTurnRequest {
9647                thread_id: "thread-ledger-checkpoint".to_string(),
9648                message: "write the answer file".to_string(),
9649                images: Vec::new(),
9650                provider_override: None,
9651                model_override: None,
9652                reasoning_override: None,
9653                workspace: test_workspace(),
9654                instructions: InstructionBundle::default(),
9655                developer_context: None,
9656                task_ledger_required: true,
9657            })
9658            .await
9659            .unwrap();
9660
9661        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9662            loop {
9663                let envelope = rx.recv().await.unwrap();
9664                if envelope.turn_id.as_deref() != Some(&turn_id) {
9665                    continue;
9666                }
9667                match envelope.event {
9668                    RoderEvent::TurnCompleted(_) => break,
9669                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9670                    _ => {}
9671                }
9672            }
9673        })
9674        .await
9675        .unwrap();
9676
9677        let requests = requests.lock().unwrap().clone();
9678        assert!(requests.len() >= 2);
9679        assert!(requests[1].transcript.iter().any(|item| {
9680            matches!(
9681                item,
9682                TranscriptItem::UserMessage(message)
9683                    if message.text.contains("Scoreable Output Checkpoint")
9684                        && message.text.contains("ensure the required output file(s) exist")
9685                        && message.text.contains("Write /app/result.txt")
9686            )
9687        }));
9688    }
9689
9690    #[test]
9691    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9692        let prompt = task_ledger_deadline_completion_prompt(
9693            12,
9694            30,
9695            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9696        );
9697
9698        assert!(prompt.contains("12 seconds remain"));
9699        assert!(prompt.contains("create or update the required scoreable output files"));
9700        assert!(prompt.contains("write /app/result.txt"));
9701        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9702    }
9703
9704    #[test]
9705    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9706        let prompt = task_ledger_scoreable_checkpoint_prompt(
9707            120,
9708            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9709        );
9710
9711        assert!(prompt.contains("120 seconds remain"));
9712        assert!(prompt.contains("best evidence-backed answer"));
9713        assert!(prompt.contains("even if provisional"));
9714        assert!(prompt.contains("preserve that candidate"));
9715        assert!(prompt.contains("partial-coverage"));
9716        assert!(prompt.contains("write /app/result.txt"));
9717        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9718    }
9719
9720    #[test]
9721    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9722        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9723        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9724            id: "ledger-open".to_string(),
9725            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9726            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9727                .to_string(),
9728            display_payload: None,
9729            is_error: false,
9730        })];
9731
9732        let (_, action) = inference_timeout_deadline(
9733            deadline,
9734            RuntimeProfile::Eval,
9735            true,
9736            30,
9737            false,
9738            0,
9739            &transcript,
9740        )
9741        .unwrap();
9742
9743        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9744    }
9745
9746    #[tokio::test]
9747    async fn verification_gate_forces_eval_code_changes_through_review() {
9748        let mut builder = ExtensionRegistryBuilder::new();
9749        builder.inference_engine(Arc::new(VerificationGateEngine {
9750            calls: StdMutex::new(0),
9751        }));
9752        builder.tool_contributor(Arc::new(WriteFileContributor));
9753        builder.tool_contributor(Arc::new(
9754            roder_ext_verification::VerificationToolContributor,
9755        ));
9756        let runtime = Arc::new(
9757            Runtime::new(
9758                builder.build().unwrap(),
9759                RuntimeConfig {
9760                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9761                    default_model: "mock".to_string(),
9762                    runtime_profile: RuntimeProfile::Eval,
9763                    policy_mode: PolicyMode::Bypass,
9764                    agent_swarm_mode: false,
9765                    ultra_mode: false,
9766                    ..RuntimeConfig::default()
9767                },
9768            )
9769            .unwrap(),
9770        );
9771        let mut rx = runtime.subscribe_events();
9772        let turn_id = runtime
9773            .start_turn(StartTurnRequest {
9774                thread_id: "thread-verification".to_string(),
9775                message: "write code".to_string(),
9776                images: Vec::new(),
9777                provider_override: None,
9778                model_override: None,
9779                reasoning_override: None,
9780                workspace: test_workspace(),
9781                instructions: InstructionBundle::default(),
9782                developer_context: None,
9783                task_ledger_required: false,
9784            })
9785            .await
9786            .unwrap();
9787
9788        let mut saw_required = false;
9789        let mut saw_completed = false;
9790        let mut final_text = String::new();
9791        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9792            loop {
9793                let envelope = rx.recv().await.unwrap();
9794                if envelope.turn_id.as_deref() != Some(&turn_id) {
9795                    continue;
9796                }
9797                match envelope.event {
9798                    RoderEvent::VerificationRequired(event) => {
9799                        saw_required = true;
9800                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9801                    }
9802                    RoderEvent::VerificationCompleted(event) => {
9803                        saw_completed = true;
9804                        assert!(event.passed);
9805                    }
9806                    RoderEvent::InferenceEventReceived(event) => {
9807                        if let InferenceEvent::MessageDelta(delta) = event.event {
9808                            final_text.push_str(&delta.text);
9809                        }
9810                    }
9811                    RoderEvent::TurnCompleted(_) => break,
9812                    _ => {}
9813                }
9814            }
9815        })
9816        .await
9817        .unwrap();
9818
9819        assert!(saw_required);
9820        assert!(saw_completed);
9821        assert!(final_text.contains("verified final"));
9822    }
9823
9824    #[tokio::test]
9825    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9826        let requests = Arc::new(StdMutex::new(Vec::new()));
9827        let mut builder = ExtensionRegistryBuilder::new();
9828        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9829            calls: StdMutex::new(0),
9830            requests: requests.clone(),
9831        }));
9832        builder.tool_contributor(Arc::new(WriteFileContributor));
9833        builder.tool_contributor(Arc::new(
9834            roder_ext_verification::VerificationToolContributor,
9835        ));
9836        let runtime = Arc::new(
9837            Runtime::new(
9838                builder.build().unwrap(),
9839                RuntimeConfig {
9840                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9841                    default_model: "gpt-5.5".to_string(),
9842                    runtime_profile: RuntimeProfile::Eval,
9843                    policy_mode: PolicyMode::Bypass,
9844                    agent_swarm_mode: false,
9845                    ultra_mode: false,
9846                    ..RuntimeConfig::default()
9847                },
9848            )
9849            .unwrap(),
9850        );
9851        let mut rx = runtime.subscribe_events();
9852        let turn_id = runtime
9853            .start_turn(StartTurnRequest {
9854                thread_id: "thread-speed-policy".to_string(),
9855                message: "write code".to_string(),
9856                images: Vec::new(),
9857                provider_override: None,
9858                model_override: None,
9859                reasoning_override: None,
9860                workspace: test_workspace(),
9861                instructions: InstructionBundle::default(),
9862                developer_context: None,
9863                task_ledger_required: false,
9864            })
9865            .await
9866            .unwrap();
9867
9868        let mut saw_speed_policy_event = false;
9869        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9870            loop {
9871                let envelope = rx.recv().await.unwrap();
9872                if envelope.turn_id.as_deref() != Some(&turn_id) {
9873                    continue;
9874                }
9875                match envelope.event {
9876                    RoderEvent::InferenceStarted(event) => {
9877                        if event.speed_policy.is_some() {
9878                            saw_speed_policy_event = true;
9879                        }
9880                    }
9881                    RoderEvent::TurnCompleted(_) => break,
9882                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9883                    _ => {}
9884                }
9885            }
9886        })
9887        .await
9888        .unwrap();
9889
9890        let requests = requests.lock().unwrap().clone();
9891        assert!(saw_speed_policy_event);
9892        assert!(requests.len() >= 4);
9893        assert!(requests.iter().all(|request| {
9894            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9895                && request.model.model == "gpt-5.5"
9896        }));
9897        assert_eq!(
9898            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9899            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9900        );
9901        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9902        assert_eq!(
9903            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9904            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9905        );
9906        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9907        assert_eq!(
9908            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9909            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9910        );
9911        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9912        assert_eq!(
9913            requests[2]
9914                .metadata
9915                .pointer("/speedPolicy/phase")
9916                .and_then(serde_json::Value::as_str),
9917            Some("verification")
9918        );
9919    }
9920
9921    #[tokio::test]
9922    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9923        let mut builder = ExtensionRegistryBuilder::new();
9924        builder.inference_engine(Arc::new(DeadlineEngine));
9925        let runtime = Arc::new(
9926            Runtime::new(
9927                builder.build().unwrap(),
9928                RuntimeConfig {
9929                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9930                    default_model: "mock".to_string(),
9931                    runtime_profile: RuntimeProfile::Eval,
9932                    turn_deadline_seconds: Some(1),
9933                    ..RuntimeConfig::default()
9934                },
9935            )
9936            .unwrap(),
9937        );
9938        let mut rx = runtime.subscribe_events();
9939        let turn_id = runtime
9940            .start_turn(StartTurnRequest {
9941                thread_id: "thread-deadline".to_string(),
9942                message: "slow work".to_string(),
9943                images: Vec::new(),
9944                provider_override: None,
9945                model_override: None,
9946                reasoning_override: None,
9947                workspace: test_workspace(),
9948                instructions: InstructionBundle::default(),
9949                developer_context: None,
9950                task_ledger_required: false,
9951            })
9952            .await
9953            .unwrap();
9954
9955        let mut saw_partial = false;
9956        let mut saw_deadline = false;
9957        let mut failed_kind = None;
9958        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9959            loop {
9960                let envelope = rx.recv().await.unwrap();
9961                if envelope.turn_id.as_deref() != Some(&turn_id) {
9962                    continue;
9963                }
9964                match envelope.event {
9965                    RoderEvent::TurnPartialResult(event) => {
9966                        saw_partial = event.summary.contains("partial turn state");
9967                    }
9968                    RoderEvent::TurnDeadlineExceeded(event) => {
9969                        saw_deadline = event.partial_result.contains("transcript items");
9970                    }
9971                    RoderEvent::TurnFailed(event) => {
9972                        failed_kind = event.error_kind;
9973                        break;
9974                    }
9975                    _ => {}
9976                }
9977            }
9978        })
9979        .await
9980        .unwrap();
9981
9982        for _ in 0..20 {
9983            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9984                break;
9985            }
9986            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9987        }
9988        assert!(saw_partial);
9989        assert!(saw_deadline);
9990        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9991        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9992    }
9993
9994    #[tokio::test]
9995    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9996        let calls = Arc::new(StdMutex::new(0));
9997        let mut builder = ExtensionRegistryBuilder::new();
9998        builder.inference_engine(Arc::new(CapturingEngine {
9999            request: Arc::new(StdMutex::new(None)),
10000        }));
10001        let task_tool = Arc::new(CountingTaskTool {
10002            calls: calls.clone(),
10003        });
10004        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10005        let runtime = Arc::new(
10006            Runtime::new(
10007                builder.build().unwrap(),
10008                RuntimeConfig {
10009                    policy_mode: PolicyMode::Bypass,
10010                    agent_swarm_mode: false,
10011                    ultra_mode: false,
10012                    ..RuntimeConfig::default()
10013                },
10014            )
10015            .unwrap(),
10016        );
10017
10018        let result = runtime
10019            .route_tool_call(
10020                &"thread-deadline-task".to_string(),
10021                &"turn-deadline-task".to_string(),
10022                ToolCallCompleted {
10023                    id: "task-1".to_string(),
10024                    name: "task".to_string(),
10025                    arguments: serde_json::json!({
10026                        "description": "inspect",
10027                        "prompt": "read"
10028                    })
10029                    .to_string(),
10030                },
10031                None,
10032                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10033            )
10034            .await
10035            .unwrap();
10036
10037        assert!(result.is_error);
10038        assert!(result.result.contains("deadline policy skipped"));
10039        assert_eq!(*calls.lock().unwrap(), 0);
10040    }
10041
10042    struct TestToolContributor {
10043        tool: Arc<dyn ToolExecutor>,
10044    }
10045
10046    impl ToolContributor for TestToolContributor {
10047        fn id(&self) -> String {
10048            "test-tool".to_string()
10049        }
10050
10051        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10052            registry.register(self.tool.clone())
10053        }
10054    }
10055
10056    #[tokio::test]
10057    async fn agent_swarm_mode_override_is_per_thread() {
10058        let runtime = Runtime::fake().unwrap();
10059        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10060
10061        // Off everywhere by default.
10062        assert!(
10063            !runtime
10064                .effective_agent_swarm_mode_for_thread("thread-a")
10065                .await
10066        );
10067        assert!(
10068            !runtime
10069                .effective_agent_swarm_mode_for_thread("thread-b")
10070                .await
10071        );
10072
10073        // Enabling on thread-a does not leak into thread-b.
10074        assert!(
10075            runtime
10076                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10077                .await
10078        );
10079        assert!(
10080            runtime
10081                .effective_agent_swarm_mode_for_thread("thread-a")
10082                .await
10083        );
10084        assert!(
10085            !runtime
10086                .effective_agent_swarm_mode_for_thread("thread-b")
10087                .await
10088        );
10089
10090        // A per-thread `off` override wins over a runtime-global `on` default.
10091        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10092        runtime
10093            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10094            .await;
10095        assert!(
10096            !runtime
10097                .effective_agent_swarm_mode_for_thread("thread-b")
10098                .await,
10099            "explicit per-thread off overrides the global on default"
10100        );
10101        // A thread with no override still follows the global default.
10102        assert!(
10103            runtime
10104                .effective_agent_swarm_mode_for_thread("thread-c")
10105                .await,
10106            "threads without an override follow the runtime-global default"
10107        );
10108    }
10109
10110    #[tokio::test]
10111    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10112        let runtime = Runtime::fake().unwrap();
10113        let mut events = runtime.subscribe_events();
10114        runtime
10115            .set_agent_swarm_mode_for_thread(
10116                "thread-xyz",
10117                true,
10118                roder_api::subagents::AgentSwarmModeTrigger::Task,
10119            )
10120            .await;
10121        let mut saw = false;
10122        for _ in 0..8 {
10123            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10124                .await
10125                .unwrap()
10126                .unwrap();
10127            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10128                assert_eq!(event.thread_id, "thread-xyz");
10129                assert!(event.enabled);
10130                assert_eq!(
10131                    event.trigger,
10132                    roder_api::subagents::AgentSwarmModeTrigger::Task
10133                );
10134                saw = true;
10135                break;
10136            }
10137        }
10138        assert!(
10139            saw,
10140            "expected an AgentSwarmModeChanged event for the thread"
10141        );
10142    }
10143
10144    #[tokio::test]
10145    async fn ultra_mode_override_is_per_thread() {
10146        let runtime = Runtime::fake().unwrap();
10147        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10148
10149        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10150        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10151
10152        assert!(
10153            runtime
10154                .set_ultra_mode_for_thread("thread-a", true, trigger)
10155                .await
10156        );
10157        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10158        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10159
10160        runtime.set_ultra_mode(true, trigger).await.unwrap();
10161        runtime
10162            .set_ultra_mode_for_thread("thread-b", false, trigger)
10163            .await;
10164        assert!(
10165            !runtime.effective_ultra_mode_for_thread("thread-b").await,
10166            "explicit per-thread off overrides the global on default"
10167        );
10168        assert!(
10169            runtime.effective_ultra_mode_for_thread("thread-c").await,
10170            "threads without an override follow the runtime-global default"
10171        );
10172    }
10173
10174    #[tokio::test]
10175    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10176        let runtime = Runtime::fake().unwrap();
10177        let mut events = runtime.subscribe_events();
10178        runtime
10179            .set_ultra_mode_for_thread(
10180                "thread-ultra",
10181                true,
10182                roder_api::subagents::UltraModeTrigger::Task,
10183            )
10184            .await;
10185        let mut saw = false;
10186        for _ in 0..8 {
10187            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10188                .await
10189                .unwrap()
10190                .unwrap();
10191            if let RoderEvent::UltraModeChanged(event) = envelope.event {
10192                assert_eq!(event.thread_id, "thread-ultra");
10193                assert!(event.enabled);
10194                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10195                saw = true;
10196                break;
10197            }
10198        }
10199        assert!(saw, "expected an UltraModeChanged event for the thread");
10200    }
10201}