Skip to main content

roder_core/
runtime.rs

1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12    model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33    ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37    RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42    TeamMemberStatus,
43};
44use roder_api::thread::{
45    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51    TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67    classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73    route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77    apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78    apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83    provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
94/// limit into a continuation, or as the empty-tool-call persistence nudge, so
95/// the model keeps working instead of ending the turn early.
96const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
98/// its render loop, which during an active turn only wakes at the ~6 FPS status
99/// animation cadence (backend events do not wake the input poll), so up to
100/// ~166ms of events buffer between drains. A reasoning-high provider that runs
101/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
102/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
103/// 1024-slot buffer overflowed in those windows and silently dropped tool and
104/// thinking rows from the live view. Sized for generous headroom across bursts
105/// and brief render stalls.
106const EVENT_BUS_CAPACITY: usize = 16_384;
107const FINAL_ANSWER_PHASE: &str = "final_answer";
108pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
109const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
110const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
111const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
112pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
113const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
114const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
115const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
116
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum InferenceTimeoutAction {
119    ScoreableCheckpoint,
120    Finalization,
121}
122
123#[derive(Debug, Clone)]
124pub struct RuntimeConfig {
125    pub default_provider: String,
126    pub default_model: String,
127    pub reasoning: Option<String>,
128    pub auto_compact_token_limit: Option<u32>,
129    pub file_backed_dynamic_context: bool,
130    pub hosted_web_search: HostedWebSearchConfig,
131    pub tool_search: ToolSearchConfig,
132    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
133    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
134    pub model_edit_tools: HashMap<String, String>,
135    pub model_parallel_tool_calls: HashMap<String, bool>,
136    pub model_profiles: HashMap<String, ModelHarnessProfile>,
137    pub tool_allowlist: Vec<String>,
138    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
139    pub external_tool_timeout_seconds: u64,
140    pub command_shell: String,
141    pub workspace: Option<String>,
142    pub policy_mode: PolicyMode,
143    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
144    /// injects the swarm reminder into each turn's developer instructions so any
145    /// client benefits, not just the TUI.
146    pub agent_swarm_mode: bool,
147    pub runtime_profile: RuntimeProfile,
148    pub inference_router: RuntimeInferenceRouterConfig,
149    pub speed_policy: RuntimeSpeedPolicyConfig,
150    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
151    pub reliability: RuntimeReliabilityConfig,
152    pub turn_deadline_seconds: Option<u64>,
153    pub remote_runner_destination: Option<RunnerDestination>,
154    pub team_data_dir: Option<PathBuf>,
155    pub roadmap_data_dir: Option<PathBuf>,
156    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
157}
158
159impl Default for RuntimeConfig {
160    fn default() -> Self {
161        Self {
162            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
163            default_model: "mock".to_string(),
164            reasoning: None,
165            auto_compact_token_limit: None,
166            file_backed_dynamic_context: true,
167            hosted_web_search: HostedWebSearchConfig::cached(),
168            tool_search: ToolSearchConfig::default(),
169            provider_tool_search: HashMap::new(),
170            model_tool_search: HashMap::new(),
171            model_edit_tools: HashMap::new(),
172            model_parallel_tool_calls: HashMap::new(),
173            model_profiles: HashMap::new(),
174            tool_allowlist: Vec::new(),
175            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
176            command_shell: roder_api::command_shell::default_command_shell(),
177            workspace: None,
178            policy_mode: PolicyMode::Default,
179            agent_swarm_mode: false,
180            runtime_profile: RuntimeProfile::Interactive,
181            inference_router: RuntimeInferenceRouterConfig::default(),
182            speed_policy: RuntimeSpeedPolicyConfig::default(),
183            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
184            reliability: RuntimeReliabilityConfig::default(),
185            turn_deadline_seconds: None,
186            remote_runner_destination: None,
187            team_data_dir: None,
188            roadmap_data_dir: None,
189            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
190        }
191    }
192}
193
194#[derive(Debug, Clone)]
195pub struct StartTurnRequest {
196    pub thread_id: ThreadId,
197    pub message: String,
198    pub images: Vec<InputImage>,
199    pub provider_override: Option<String>,
200    pub model_override: Option<String>,
201    pub reasoning_override: Option<String>,
202    pub workspace: String,
203    pub instructions: InstructionBundle,
204    /**
205     * Per-turn developer-authority context for this turn's InstructionBundle.
206     * Applies to every inference round of the turn, is never written to
207     * thread state, and does not carry over to later turns.
208     */
209    pub developer_context: Option<String>,
210    pub task_ledger_required: bool,
211}
212
213#[derive(Debug, Clone)]
214pub struct CreateThreadRequest {
215    pub title: Option<String>,
216    pub workspace: String,
217    pub workspace_id: Option<String>,
218    pub root_id: Option<String>,
219    pub provider: Option<String>,
220    pub model: Option<String>,
221    pub selection_mode: Option<ModelSelectionMode>,
222    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
223    pub tool_allowlist: Vec<String>,
224    /// Host-supplied instructions added to the developer slot of every turn's inference request.
225    pub developer_instructions: Option<String>,
226    /// Host-executed tool specs advertised to the model on every turn of this thread.
227    pub external_tools: Vec<roder_api::tools::ToolSpec>,
228    /// Explicit remote-runner binding for the thread's native coding tools.
229    pub runner: Option<ThreadRunnerSelection>,
230}
231
232/**
233 * Thread-level remote-runner selection. The destination config is persisted
234 * with the thread, so secrets must reach the provider through its
235 * environment, not this config.
236 */
237#[derive(Debug, Clone)]
238pub struct ThreadRunnerSelection {
239    pub provider_id: String,
240    pub config: serde_json::Value,
241    /// Absolute path on the runner used as the thread's coding-tool workspace root.
242    pub workspace: String,
243    /**
244     * Extra absolute runner paths file reads may resolve under, beyond
245     * `workspace`. Writes and the working directory stay confined to
246     * `workspace`.
247     */
248    pub read_roots: Vec<String>,
249}
250
251#[derive(Debug, Clone, PartialEq, Eq)]
252pub struct PendingPlanExit {
253    pub thread_id: ThreadId,
254    pub turn_id: TurnId,
255    pub request_id: String,
256    pub target_mode: PolicyMode,
257    pub plan_summary: Option<String>,
258    pub next_steps: Vec<String>,
259    pub requested_at: OffsetDateTime,
260    pub expires_at: Option<OffsetDateTime>,
261}
262
263pub(crate) struct PendingToolApproval {
264    pub(crate) thread_id: ThreadId,
265    pub(crate) turn_id: TurnId,
266    pub(crate) tool_id: String,
267    pub(crate) tool_name: String,
268    pub(crate) call: roder_api::tools::ToolCall,
269    pub(crate) tx: oneshot::Sender<bool>,
270}
271
272pub(crate) struct PendingUserInput {
273    pub(crate) thread_id: ThreadId,
274    pub(crate) turn_id: TurnId,
275    pub(crate) tx: oneshot::Sender<serde_json::Value>,
276}
277
278/// Host answer to an external tool call delivered via `tools/resolve`.
279#[derive(Debug, Clone, PartialEq, Eq)]
280pub struct ExternalToolResolution {
281    pub output: String,
282    pub is_error: bool,
283}
284
285pub(crate) struct PendingExternalToolCall {
286    pub(crate) thread_id: ThreadId,
287    pub(crate) turn_id: TurnId,
288    pub(crate) tool_id: String,
289    pub(crate) tool_name: String,
290    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
291}
292
293#[derive(Clone)]
294struct ActiveTurnHandle {
295    thread_id: ThreadId,
296    abort: AbortHandle,
297    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
298    drain: Arc<TurnDrainHandle>,
299}
300
301/// Tracks a task after it has been interrupted and removed from the interactive
302/// active-turn map. This lets users start a follow-up turn immediately while a
303/// bounded runtime shutdown can still await the original task's cleanup.
304struct TurnDrainHandle {
305    thread_id: ThreadId,
306    interrupt_requested: AtomicBool,
307    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
308    completed: AtomicBool,
309    completed_notify: Notify,
310}
311
312/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
313/// tasks have reached their cleanup paths; provider-specific child-process
314/// reaping remains an explicit provider capability.
315#[derive(Debug, Clone, PartialEq, Eq)]
316pub enum RuntimeDrainOutcome {
317    Clean {
318        interrupted_turn_ids: Vec<TurnId>,
319    },
320    DeadlineExceeded {
321        interrupted_turn_ids: Vec<TurnId>,
322        remaining_turn_ids: Vec<TurnId>,
323    },
324    /// At least one lifecycle transition initiated by this drain could not be
325    /// durably appended. The runtime still made its best cleanup attempt, but
326    /// callers must not claim a persisted terminal state as proof of recovery.
327    PersistenceFailed {
328        interrupted_turn_ids: Vec<TurnId>,
329        remaining_turn_ids: Vec<TurnId>,
330    },
331}
332
333#[derive(Clone)]
334struct QueuedTurnSteer {
335    message: UserMessage,
336    mailbox_ack: Option<MailboxDeliveryAck>,
337}
338
339#[derive(Clone)]
340struct MailboxDeliveryAck {
341    team_id: TeamId,
342    message_ids: Vec<String>,
343}
344
345#[derive(Clone)]
346struct InheritedTurnContext {
347    workspace: String,
348    instructions: InstructionBundle,
349    developer_context: Option<String>,
350}
351
352#[derive(Debug, Clone, Default, PartialEq, Eq)]
353pub struct ThreadActivity {
354    pub active_turn_id: Option<TurnId>,
355    pub active_flags: Vec<String>,
356}
357
358/// Per-thread settings persisted at thread creation and applied to every turn.
359#[derive(Debug, Clone, Default)]
360pub(crate) struct ThreadTurnOverrides {
361    pub(crate) tool_allowlist: Vec<String>,
362    pub(crate) developer_instructions: Option<String>,
363    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
364}
365
366#[derive(Debug, Clone, Copy, PartialEq, Eq)]
367pub(crate) enum TurnRunOutcome {
368    Completed,
369    Stopped,
370}
371
372impl PendingPlanExit {
373    pub fn new(
374        thread_id: ThreadId,
375        turn_id: TurnId,
376        request_id: String,
377        target_mode: PolicyMode,
378        plan_summary: Option<String>,
379        next_steps: Vec<String>,
380    ) -> Self {
381        let requested_at = OffsetDateTime::now_utc();
382        Self {
383            thread_id,
384            turn_id,
385            request_id,
386            target_mode,
387            plan_summary,
388            next_steps,
389            requested_at,
390            expires_at: Some(requested_at + default_plan_exit_timeout()),
391        }
392    }
393
394    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
395        self.expires_at.is_some_and(|expires_at| now >= expires_at)
396    }
397}
398
399pub fn default_plan_exit_timeout() -> Duration {
400    Duration::minutes(10)
401}
402
403pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
404
405fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
406    messages
407        .iter()
408        .map(|message| {
409            let recipient = team
410                .members
411                .iter()
412                .find(|member| member.id == message.to_member_id)
413                .map(canonical_team_member_path)
414                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
415            let sender = message
416                .from_member_id
417                .as_deref()
418                .and_then(|id| team.members.iter().find(|member| member.id == id))
419                .map(canonical_team_member_path)
420                .unwrap_or_else(|| "/root".to_string());
421            let message_type = match message.kind {
422                TeamMailboxMessageKind::Message => "MESSAGE",
423                TeamMailboxMessageKind::NewTask => "NEW_TASK",
424                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
425            };
426            format!(
427                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
428                message.text
429            )
430        })
431        .collect::<Vec<_>>()
432        .join("\n\n")
433}
434
435fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
436    if let Some(agent_path) = member
437        .agent_path
438        .as_deref()
439        .filter(|path| *path == "/root" || path.starts_with("/root/"))
440    {
441        return agent_path.to_string();
442    }
443    if member.role == TeamMemberRole::Lead {
444        return "/root".to_string();
445    }
446    member
447        .task_name
448        .as_deref()
449        .filter(|name| !name.trim().is_empty())
450        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
451        .unwrap_or_else(|| format!("/root/{}", member.id))
452}
453
454fn is_codex_v2_team(team: &TeamState) -> bool {
455    team.members.iter().any(|member| {
456        member
457            .model
458            .as_deref()
459            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
460    })
461}
462
463fn runtime_local_team_view(
464    mut team: TeamState,
465    active_thread_ids: &std::collections::HashSet<ThreadId>,
466) -> TeamState {
467    for member in &mut team.members {
468        if member.status == TeamMemberStatus::Running
469            && !active_thread_ids.contains(&member.thread_id)
470        {
471            member.status = TeamMemberStatus::Interrupted;
472            member.current_turn_id = None;
473        }
474    }
475    team
476}
477
478type RunnerToolExecutionKey = (String, String);
479type RunnerToolExecutionMutex = Mutex<()>;
480type RunnerToolExecutionLockRegistry =
481    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
482
483pub struct Runtime {
484    pub bus: EventBus,
485    pub registry: ExtensionRegistry,
486    config: RwLock<RuntimeConfig>,
487    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
488    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
489    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
490    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
491    /// Serializes turn admission with shutdown quiescing. A drain takes this
492    /// gate before flipping `accepting_turns` and snapshotting active work so a
493    /// turn that observed the old flag cannot be inserted after the snapshot.
494    turn_admission: Mutex<()>,
495    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
496    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
497    /// Provider cleanup handles register synchronously through the inference
498    /// tool-executor context. They are intentionally separate from active turn
499    /// admission so an interrupted turn can remain drainable after a follow-up
500    /// turn starts on the same thread.
501    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
502    accepting_turns: AtomicBool,
503    /// Monotonic counter used by bounded drains to surface lifecycle-state
504    /// persistence failures without making an individual provider turn fail.
505    lifecycle_persistence_failures: AtomicUsize,
506    lifecycle_shutdown_drains: AtomicUsize,
507    lifecycle_clean_shutdowns: AtomicUsize,
508    lifecycle_deadline_exceeded: AtomicUsize,
509    lifecycle_persistence_failed_drains: AtomicUsize,
510    lifecycle_restart_reconciliations: AtomicUsize,
511    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
512    provider_cleanup_confirmed: AtomicUsize,
513    provider_cleanup_timed_out: AtomicUsize,
514    provider_cleanup_unknown: AtomicUsize,
515    active_turns_changed: Notify,
516    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
517    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
518    /// Process-local execution boundary. Local/CLI runtimes default to true;
519    /// hosted runtime pools set this false so missing runner metadata fails
520    /// closed instead of exposing the host workspace.
521    allow_local_workspaces: AtomicBool,
522    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
523    // This stays process-local because developer_context is explicitly volatile and must never
524    // be persisted to thread state.
525    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
526    workspace: PathBuf,
527    teams: TeamManager,
528    agent_team_spawn_lock: Mutex<()>,
529    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
530    pub(crate) goals: Arc<RuntimeGoalController>,
531    context_artifacts: roder_api::artifacts::ContextArtifactStore,
532    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
533    thread_item_cache: Mutex<ThreadItemCache>,
534    pub(crate) tool_registry: ToolRegistry,
535    media_generation: Arc<crate::media_generation::MediaGenerationService>,
536    pub(crate) skills: RwLock<SkillRegistry>,
537    /// Lazily-started bounded dispatch of emitted events to registry
538    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
539    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
540    pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
541    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
542    /// this map uses its stored value; absent threads fall back to the
543    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
544    /// team per-member policy-mode override idiom so swarm mode is per-thread
545    /// like the other `thread/*` operations, without a separate runtime.
546    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
547    /**
548     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
549     * held across provider initialization, making the first workspace-tool
550     * access a singleflight while allowing unrelated threads to initialize in
551     * parallel. Failed initialization is shared with current waiters, then
552     * evicted so a later tool call can retry.
553     */
554    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
555    /**
556     * Outer tool-call locks keyed by the actual remote session, not the
557     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
558     * serializes threads that share one sandbox without coupling different
559     * tenants or independent runner sessions.
560     */
561    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
562}
563
564#[derive(Clone)]
565struct CachedRunnerSession {
566    destination: RunnerDestination,
567    session: Arc<dyn RemoteRunnerSession>,
568}
569
570struct RunnerSessionSlot {
571    provider_id: String,
572    destination_id: String,
573    state: Mutex<RunnerSessionSlotState>,
574    initialized: Notify,
575}
576
577enum RunnerSessionSlotState {
578    Empty,
579    Initializing,
580    Ready(CachedRunnerSession),
581    Failed(Arc<str>),
582}
583
584impl RunnerSessionSlot {
585    fn empty(destination: &RunnerDestination) -> Self {
586        Self {
587            provider_id: destination.provider_id.clone(),
588            destination_id: destination.id.clone(),
589            state: Mutex::new(RunnerSessionSlotState::Empty),
590            initialized: Notify::new(),
591        }
592    }
593
594    fn ready(session: CachedRunnerSession) -> Self {
595        Self {
596            provider_id: session.destination.provider_id.clone(),
597            destination_id: session.destination.id.clone(),
598            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
599            initialized: Notify::new(),
600        }
601    }
602
603    fn matches(&self, destination: &RunnerDestination) -> bool {
604        self.provider_id == destination.provider_id && self.destination_id == destination.id
605    }
606}
607
608impl Runtime {
609    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
610        if registry.inference_engines.is_empty() {
611            anyhow::bail!("at least one inference engine must be registered");
612        }
613        validate_runtime_config_reasoning(&config)?;
614        validate_runtime_inference_router_config(&registry, &config)?;
615
616        let bus = EventBus::new(EVENT_BUS_CAPACITY);
617        let thread_store = registry
618            .thread_stores
619            .first()
620            .map(|factory| factory.create());
621        let mut tool_registry = ToolRegistry::default();
622        for contributor in &registry.tools {
623            contributor
624                .contribute(&mut tool_registry)
625                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
626        }
627        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
628
629        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
630            registry.media_generator_providers.clone(),
631            config.media_generation.clone(),
632        ));
633        tool_registry.replace(Arc::new(
634            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
635        ));
636
637        let team_data_dir = config.team_data_dir.clone();
638        let workspace = config
639            .workspace
640            .clone()
641            .map(PathBuf::from)
642            .unwrap_or(std::env::current_dir()?);
643        let roadmap_data_dir = config
644            .roadmap_data_dir
645            .clone()
646            .unwrap_or_else(|| workspace.join(".roder"));
647        let context_artifacts = thread_store
648            .as_ref()
649            .and_then(|store| store.context_artifact_store())
650            .or_else(|| {
651                thread_store
652                    .as_ref()
653                    .and_then(|store| store.local_thread_root())
654                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
655            })
656            .unwrap_or_else(|| {
657                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
658            });
659        let goals = Arc::new(RuntimeGoalController::new(
660            bus.clone(),
661            thread_store.clone(),
662        ));
663        let runtime = Self {
664            bus,
665            registry,
666            config: RwLock::new(config),
667            pending_plan_exit: RwLock::new(None),
668            pending_tool_approvals: Mutex::new(HashMap::new()),
669            pending_user_inputs: Mutex::new(HashMap::new()),
670            pending_external_tool_calls: Mutex::new(HashMap::new()),
671            turn_admission: Mutex::new(()),
672            active_turns: RwLock::new(HashMap::new()),
673            turn_drains: RwLock::new(HashMap::new()),
674            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
675            accepting_turns: AtomicBool::new(true),
676            lifecycle_persistence_failures: AtomicUsize::new(0),
677            lifecycle_shutdown_drains: AtomicUsize::new(0),
678            lifecycle_clean_shutdowns: AtomicUsize::new(0),
679            lifecycle_deadline_exceeded: AtomicUsize::new(0),
680            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
681            lifecycle_restart_reconciliations: AtomicUsize::new(0),
682            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
683            provider_cleanup_confirmed: AtomicUsize::new(0),
684            provider_cleanup_timed_out: AtomicUsize::new(0),
685            provider_cleanup_unknown: AtomicUsize::new(0),
686            active_turns_changed: Notify::new(),
687            active_turn_selections: RwLock::new(HashMap::new()),
688            active_turn_contexts: RwLock::new(HashMap::new()),
689            allow_local_workspaces: AtomicBool::new(true),
690            team_member_turn_contexts: Mutex::new(HashMap::new()),
691            workspace: workspace.clone(),
692            teams: TeamManager::new(
693                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
694            ),
695            agent_team_spawn_lock: Mutex::new(()),
696            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
697                workspace,
698                roadmap_data_dir,
699            )),
700            goals,
701            context_artifacts,
702            thread_store,
703            thread_item_cache: Mutex::new(ThreadItemCache::default()),
704            tool_registry,
705            media_generation,
706            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
707                PathBuf::new(),
708            ))),
709            event_sink_dispatcher: tokio::sync::OnceCell::new(),
710            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
711            agent_swarm_modes: RwLock::new(HashMap::new()),
712            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
713            runner_tool_execution_locks: Mutex::new(HashMap::new()),
714        };
715        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
716            timestamp: OffsetDateTime::now_utc(),
717        }));
718        for manifest in &runtime.registry.manifests {
719            runtime
720                .bus
721                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
722                    extension_id: manifest.id.clone(),
723                    timestamp: OffsetDateTime::now_utc(),
724                }));
725        }
726        Ok(runtime)
727    }
728
729    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
730        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
731        builder.inference_engine(engine);
732        Self::new(builder.build()?, RuntimeConfig::default())
733    }
734
735    pub fn fake() -> anyhow::Result<Self> {
736        Self::from_engine(Arc::new(FakeInferenceEngine))
737    }
738
739    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
740        self.bus.subscribe()
741    }
742
743    /// Returns process-local, redacted lifecycle health counters. The snapshot
744    /// intentionally has fixed fields and contains no provider, command,
745    /// process, thread, or turn identifiers.
746    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
747        LifecycleMetricsSnapshot {
748            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
749            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
750            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
751                as u64,
752            persistence_failed_count: self
753                .lifecycle_persistence_failed_drains
754                .load(Ordering::Acquire) as u64,
755            restart_reconciliation_count: self
756                .lifecycle_restart_reconciliations
757                .load(Ordering::Acquire) as u64,
758            lifecycle_persistence_failure_count: self
759                .lifecycle_persistence_failures
760                .load(Ordering::Acquire) as u64,
761            shutdown_drain_duration_ms_total: self
762                .lifecycle_shutdown_drain_duration_ms_total
763                .load(Ordering::Acquire) as u64,
764            provider_cleanup_confirmed_count: self
765                .provider_cleanup_confirmed
766                .load(Ordering::Acquire) as u64,
767            provider_cleanup_timed_out_count: self
768                .provider_cleanup_timed_out
769                .load(Ordering::Acquire) as u64,
770            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
771                as u64,
772        }
773    }
774
775    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
776        let Some(store) = &self.thread_store else {
777            return true;
778        };
779        let state = match record.extension_state() {
780            Ok(state) => state,
781            Err(_) => {
782                self.lifecycle_persistence_failures
783                    .fetch_add(1, Ordering::AcqRel);
784                return false;
785            }
786        };
787        // Lifecycle diagnostics must not turn a provider result into a failed
788        // turn merely because a third-party store lacks extension-state support.
789        // JSONL/Postgres stores persist this append-only record atomically at
790        // their own storage boundary.
791        if store
792            .append_extension_state(&record.thread_id, &state)
793            .await
794            .is_err()
795        {
796            self.lifecycle_persistence_failures
797                .fetch_add(1, Ordering::AcqRel);
798            return false;
799        }
800        true
801    }
802
803    async fn record_turn_lifecycle(
804        &self,
805        thread_id: ThreadId,
806        turn_id: TurnId,
807        state: TurnLifecycleState,
808        cleanup: TurnCleanupState,
809        reason: Option<TurnLifecycleReason>,
810    ) -> TurnLifecycleRecord {
811        self.record_turn_lifecycle_with_ownership(
812            thread_id,
813            turn_id,
814            state,
815            cleanup,
816            reason,
817            TurnCleanupOwnership::RuntimeTaskOnly,
818        )
819        .await
820    }
821
822    async fn record_turn_lifecycle_with_ownership(
823        &self,
824        thread_id: ThreadId,
825        turn_id: TurnId,
826        state: TurnLifecycleState,
827        cleanup: TurnCleanupState,
828        reason: Option<TurnLifecycleReason>,
829        ownership: TurnCleanupOwnership,
830    ) -> TurnLifecycleRecord {
831        let record = TurnLifecycleRecord::new(
832            thread_id,
833            turn_id,
834            state,
835            cleanup,
836            reason,
837            OffsetDateTime::now_utc(),
838        )
839        .with_ownership(ownership);
840        let _ = self.persist_turn_lifecycle_record(&record).await;
841        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
842            .await;
843        record
844    }
845
846    pub(crate) fn register_provider_turn_cleanup(
847        &self,
848        turn_id: &TurnId,
849        cleanup: Arc<dyn ProviderTurnCleanup>,
850    ) {
851        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
852            cleanups.insert(turn_id.clone(), cleanup);
853        }
854    }
855
856    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
857        self.provider_turn_cleanups
858            .lock()
859            .ok()
860            .and_then(|cleanups| cleanups.get(turn_id).cloned())
861            .map(|cleanup| cleanup.ownership())
862            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
863    }
864
865    async fn await_provider_turn_cleanup(
866        &self,
867        turn_id: &TurnId,
868    ) -> (TurnCleanupState, TurnCleanupOwnership) {
869        let cleanup = self
870            .provider_turn_cleanups
871            .lock()
872            .ok()
873            .and_then(|mut cleanups| cleanups.remove(turn_id));
874        let Some(cleanup) = cleanup else {
875            return (
876                TurnCleanupState::Unknown,
877                TurnCleanupOwnership::RuntimeTaskOnly,
878            );
879        };
880        let ownership = cleanup.ownership();
881        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
882            Ok(Ok(())) => {
883                self.provider_cleanup_confirmed
884                    .fetch_add(1, Ordering::AcqRel);
885                (
886                    TurnCleanupState::Completed,
887                    TurnCleanupOwnership::ProviderCleanupConfirmed,
888                )
889            }
890            Ok(Err(_)) => {
891                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
892                (TurnCleanupState::Unknown, ownership)
893            }
894            Err(_) => {
895                self.provider_cleanup_timed_out
896                    .fetch_add(1, Ordering::AcqRel);
897                (TurnCleanupState::TimedOut, ownership)
898            }
899        }
900    }
901
902    fn record_lifecycle_drain_outcome(
903        &self,
904        started_at: tokio::time::Instant,
905        outcome: &RuntimeDrainOutcome,
906    ) {
907        self.lifecycle_shutdown_drains
908            .fetch_add(1, Ordering::AcqRel);
909        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
910            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
911            Ordering::AcqRel,
912        );
913        match outcome {
914            RuntimeDrainOutcome::Clean { .. } => {
915                self.lifecycle_clean_shutdowns
916                    .fetch_add(1, Ordering::AcqRel);
917            }
918            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
919                self.lifecycle_deadline_exceeded
920                    .fetch_add(1, Ordering::AcqRel);
921            }
922            RuntimeDrainOutcome::PersistenceFailed { .. } => {
923                self.lifecycle_persistence_failed_drains
924                    .fetch_add(1, Ordering::AcqRel);
925            }
926        }
927    }
928
929    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
930        match event {
931            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
932                event.thread_id.clone(),
933                event.turn_id.clone(),
934                TurnLifecycleState::Running,
935                TurnCleanupState::NotRequested,
936                None,
937                event.timestamp,
938            )),
939            // `run_turn` persists completed turns after it has awaited a
940            // registered provider cleanup handle. Do not let the generic event
941            // projection overwrite that acknowledgement with an unproven state.
942            RoderEvent::TurnCompleted(_) => None,
943            // Some failure paths already have an event before the turn wrapper
944            // reaches its cleanup acknowledgement. Preserve a durable fallback
945            // reason here; the wrapper appends a later record with the more
946            // precise cleanup outcome when a provider registered one.
947            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
948                event.thread_id.clone(),
949                event.turn_id.clone(),
950                TurnLifecycleState::Failed,
951                TurnCleanupState::Unknown,
952                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
953                    TurnLifecycleReason::DeadlineExceeded
954                } else {
955                    TurnLifecycleReason::ProviderFailure
956                }),
957                event.timestamp,
958            )),
959            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
960            RoderEvent::TurnLifecycleUpdated(_) => None,
961            _ => None,
962        }
963    }
964
965    pub fn registry(&self) -> &ExtensionRegistry {
966        &self.registry
967    }
968
969    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
970        self.media_generation.clone()
971    }
972
973    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
974        self.context_artifacts.clone()
975    }
976
977    pub async fn execute_workflow_tool(
978        &self,
979        thread_id: ThreadId,
980        tool_name: &str,
981        arguments: serde_json::Value,
982    ) -> anyhow::Result<ToolResult> {
983        let Some(executor) = self.tool_registry.get(tool_name) else {
984            anyhow::bail!("tool not found: {tool_name}");
985        };
986        let tool_call = ToolCall {
987            id: format!("slash-{tool_name}"),
988            name: tool_name.to_string(),
989            raw_arguments: serde_json::to_string(&arguments)?,
990            arguments,
991            thread_id: thread_id.clone(),
992            turn_id: "slash-command".to_string(),
993        };
994        let runtime_config = self.status().await;
995        let ctx = self.tool_execution_context(
996            thread_id,
997            "slash-command".to_string(),
998            runtime_config.policy_mode,
999            runtime_config.workspace.as_deref(),
1000            Some(&runtime_config.command_shell),
1001        );
1002        executor.execute(ctx, tool_call).await
1003    }
1004
1005    pub(crate) fn tool_execution_context(
1006        &self,
1007        thread_id: ThreadId,
1008        turn_id: TurnId,
1009        mode: PolicyMode,
1010        workspace: Option<&str>,
1011        command_shell: Option<&str>,
1012    ) -> ToolExecutionContext {
1013        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1014            .with_command_shell(command_shell.unwrap_or_default())
1015            .with_process_runner(Arc::new(LocalProcessRunner))
1016            .with_context_artifacts(self.context_artifacts.backend())
1017            .with_goal_controller(self.goals.clone())
1018            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1019                self.bus.clone(),
1020                self.thread_store.clone(),
1021            )))
1022            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1023                self.bus.clone(),
1024                self.thread_store.clone(),
1025            )));
1026        if let Some(workspace) = workspace {
1027            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1028        }
1029        ctx
1030    }
1031
1032    pub async fn status(&self) -> RuntimeConfig {
1033        self.config.read().await.clone()
1034    }
1035
1036    pub async fn set_skills(&self, skills: SkillRegistry) {
1037        *self.skills.write().await = skills;
1038    }
1039
1040    pub async fn skills_snapshot(&self) -> SkillRegistry {
1041        self.skills.read().await.clone()
1042    }
1043
1044    pub fn workspace(&self) -> PathBuf {
1045        self.workspace.clone()
1046    }
1047
1048    /// Controls whether native workspace tools may execute without an
1049    /// explicit remote-runner binding. Hosted runtimes disable this after
1050    /// construction; ordinary local runtimes retain the compatible default.
1051    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1052        self.allow_local_workspaces
1053            .store(allowed, Ordering::Relaxed);
1054    }
1055
1056    pub fn allows_local_workspaces(&self) -> bool {
1057        self.allow_local_workspaces.load(Ordering::Relaxed)
1058    }
1059
1060    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1061        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1062            destination_id: destination.id.clone(),
1063            provider_id: destination.provider_id.clone(),
1064            state: "configured".to_string(),
1065            session_id: None,
1066            timestamp: OffsetDateTime::now_utc(),
1067        });
1068        self.config.write().await.remote_runner_destination = destination;
1069        if let Some(lifecycle) = lifecycle {
1070            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1071        } else {
1072            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1073                destination_id: "local".to_string(),
1074                provider_id: "local".to_string(),
1075                state: "local_fallback".to_string(),
1076                session_id: None,
1077                timestamp: OffsetDateTime::now_utc(),
1078            }))
1079            .await;
1080        }
1081    }
1082
1083    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1084        let mut cfg = self.config.write().await;
1085        cfg.file_backed_dynamic_context = enabled;
1086        cfg.clone()
1087    }
1088
1089    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1090        let mut cfg = self.config.write().await;
1091        cfg.command_shell = shell;
1092        cfg.clone()
1093    }
1094
1095    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1096        let mut pending = self.pending_plan_exit.write().await;
1097        let current = pending.clone()?;
1098        if !current.is_expired(OffsetDateTime::now_utc()) {
1099            return Some(current);
1100        }
1101        *pending = None;
1102        drop(pending);
1103        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1104            .await;
1105        None
1106    }
1107
1108    pub async fn set_policy_mode(
1109        &self,
1110        mode: PolicyMode,
1111        reason: Option<String>,
1112    ) -> anyhow::Result<RuntimeConfig> {
1113        let mut cfg = self.config.write().await;
1114        let previous_mode = cfg.policy_mode;
1115        cfg.policy_mode = mode;
1116        let next = cfg.clone();
1117        drop(cfg);
1118        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1119            thread_id: "runtime".to_string(),
1120            turn_id: None,
1121            previous_mode,
1122            new_mode: mode,
1123            reason,
1124            timestamp: OffsetDateTime::now_utc(),
1125        }))
1126        .await;
1127        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1128            .await;
1129        Ok(next)
1130    }
1131
1132    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1133    /// swarm reminder is injected into each turn's developer instructions so
1134    /// every app-server/SDK client gets it, not only the TUI.
1135    pub async fn set_agent_swarm_mode(
1136        &self,
1137        enabled: bool,
1138        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1139    ) -> anyhow::Result<RuntimeConfig> {
1140        let mut cfg = self.config.write().await;
1141        cfg.agent_swarm_mode = enabled;
1142        let next = cfg.clone();
1143        drop(cfg);
1144        self.emit(RoderEvent::AgentSwarmModeChanged(
1145            roder_api::subagents::AgentSwarmModeChanged {
1146                thread_id: "runtime".to_string(),
1147                turn_id: None,
1148                enabled,
1149                trigger,
1150                timestamp: OffsetDateTime::now_utc(),
1151            },
1152        ))
1153        .await;
1154        Ok(next)
1155    }
1156
1157    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1158    /// stored per thread so toggling swarm mode on one thread does not leak the
1159    /// reminder into other threads sharing the runtime; absent threads fall back
1160    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1161    /// carries the real `thread_id`.
1162    pub async fn set_agent_swarm_mode_for_thread(
1163        &self,
1164        thread_id: &str,
1165        enabled: bool,
1166        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1167    ) -> bool {
1168        {
1169            let mut modes = self.agent_swarm_modes.write().await;
1170            modes.insert(thread_id.to_string(), enabled);
1171        }
1172        self.emit(RoderEvent::AgentSwarmModeChanged(
1173            roder_api::subagents::AgentSwarmModeChanged {
1174                thread_id: thread_id.to_string(),
1175                turn_id: None,
1176                enabled,
1177                trigger,
1178                timestamp: OffsetDateTime::now_utc(),
1179            },
1180        ))
1181        .await;
1182        enabled
1183    }
1184
1185    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1186    /// override when present, otherwise the runtime-global default. The turn loop
1187    /// uses this to decide whether to inject the swarm reminder.
1188    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1189        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1190            return enabled;
1191        }
1192        self.status().await.agent_swarm_mode
1193    }
1194
1195    pub async fn set_hosted_web_search(
1196        &self,
1197        mode: HostedWebSearchMode,
1198    ) -> anyhow::Result<RuntimeConfig> {
1199        let mut cfg = self.config.write().await;
1200        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1201        Ok(cfg.clone())
1202    }
1203
1204    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1205        let gate = DefaultPolicyGate::new();
1206        let mut pending = self.pending_tool_approvals.lock().await;
1207        let approval_ids = pending
1208            .iter()
1209            .filter_map(|(approval_id, approval)| {
1210                let ctx = ToolExecutionContext::new(
1211                    approval.thread_id.clone(),
1212                    approval.turn_id.clone(),
1213                    mode,
1214                );
1215                matches!(
1216                    gate.decide(&approval.call, mode, &ctx),
1217                    PolicyDecision::AutoApproved { .. }
1218                )
1219                .then_some(approval_id.clone())
1220            })
1221            .collect::<Vec<_>>();
1222        let approvals = approval_ids
1223            .into_iter()
1224            .filter_map(|approval_id| {
1225                pending
1226                    .remove(&approval_id)
1227                    .map(|approval| (approval_id, approval))
1228            })
1229            .collect::<Vec<_>>();
1230        drop(pending);
1231
1232        for (approval_id, approval) in approvals {
1233            let ctx = ToolExecutionContext::new(
1234                approval.thread_id.clone(),
1235                approval.turn_id.clone(),
1236                mode,
1237            );
1238            let decision = gate.decide(&approval.call, mode, &ctx);
1239            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1240                thread_id: approval.thread_id.clone(),
1241                turn_id: approval.turn_id.clone(),
1242                tool_id: approval.tool_id.clone(),
1243                tool_name: approval.tool_name.clone(),
1244                mode,
1245                decision,
1246                timestamp: OffsetDateTime::now_utc(),
1247            }))
1248            .await;
1249            if mode == PolicyMode::Bypass {
1250                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1251                    thread_id: approval.thread_id.clone(),
1252                    turn_id: approval.turn_id.clone(),
1253                    tool_id: approval.tool_id.clone(),
1254                    tool_name: approval.tool_name.clone(),
1255                    timestamp: OffsetDateTime::now_utc(),
1256                }))
1257                .await;
1258            }
1259            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1260                thread_id: approval.thread_id,
1261                turn_id: approval.turn_id,
1262                approval_id,
1263                tool_id: approval.tool_id,
1264                tool_name: approval.tool_name,
1265                approved: true,
1266                timestamp: OffsetDateTime::now_utc(),
1267            }))
1268            .await;
1269            let _ = approval.tx.send(true);
1270        }
1271    }
1272
1273    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1274        *self.pending_plan_exit.write().await = Some(pending.clone());
1275        self.emit(RoderEvent::PolicyExitPlanRequested(
1276            PolicyExitPlanRequested {
1277                thread_id: pending.thread_id,
1278                turn_id: pending.turn_id,
1279                request_id: pending.request_id,
1280                target_mode: pending.target_mode,
1281                plan_summary: pending.plan_summary,
1282                next_steps: pending.next_steps,
1283                timestamp: OffsetDateTime::now_utc(),
1284            },
1285        ))
1286        .await;
1287    }
1288
1289    pub async fn resolve_pending_plan_exit(
1290        &self,
1291        request_id: &str,
1292        approved: bool,
1293    ) -> anyhow::Result<Option<PendingPlanExit>> {
1294        let mut pending = self.pending_plan_exit.write().await;
1295        let Some(current) = pending.clone() else {
1296            return Ok(None);
1297        };
1298        if current.request_id != request_id {
1299            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1300        }
1301        *pending = None;
1302        drop(pending);
1303
1304        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1305        let resolved_mode = if approved {
1306            let mut cfg = self.config.write().await;
1307            let previous_mode = cfg.policy_mode;
1308            cfg.policy_mode = current.target_mode;
1309            drop(cfg);
1310            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1311                thread_id: current.thread_id.clone(),
1312                turn_id: Some(current.turn_id.clone()),
1313                previous_mode,
1314                new_mode: current.target_mode,
1315                reason: Some("approved plan exit".to_string()),
1316                timestamp: OffsetDateTime::now_utc(),
1317            }))
1318            .await;
1319            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1320                .await;
1321            current.target_mode
1322        } else {
1323            self.status().await.policy_mode
1324        };
1325        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1326            .await;
1327        Ok(Some(current))
1328    }
1329
1330    pub async fn resolve_tool_approval(
1331        &self,
1332        approval_id: &str,
1333        approved: bool,
1334    ) -> anyhow::Result<bool> {
1335        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1336        let Some(pending) = pending else {
1337            return Ok(false);
1338        };
1339        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1340            thread_id: pending.thread_id,
1341            turn_id: pending.turn_id,
1342            approval_id: approval_id.to_string(),
1343            tool_id: pending.tool_id,
1344            tool_name: pending.tool_name,
1345            approved,
1346            timestamp: OffsetDateTime::now_utc(),
1347        }))
1348        .await;
1349        let _ = pending.tx.send(approved);
1350        Ok(true)
1351    }
1352
1353    pub async fn request_app_server_tool_approval(
1354        &self,
1355        call: ToolCall,
1356        reason: Option<String>,
1357    ) -> anyhow::Result<bool> {
1358        let approval_id = call.id.clone();
1359        let (tx, rx) = oneshot::channel();
1360        self.pending_tool_approvals.lock().await.insert(
1361            approval_id.clone(),
1362            PendingToolApproval {
1363                thread_id: call.thread_id.clone(),
1364                turn_id: call.turn_id.clone(),
1365                tool_id: call.id.clone(),
1366                tool_name: call.name.clone(),
1367                call: call.clone(),
1368                tx,
1369            },
1370        );
1371        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1372            thread_id: call.thread_id.clone(),
1373            turn_id: call.turn_id.clone(),
1374            approval_id,
1375            tool_id: call.id.clone(),
1376            tool_name: call.name.clone(),
1377            reason,
1378            timestamp: OffsetDateTime::now_utc(),
1379        }))
1380        .await;
1381        Ok(rx.await.unwrap_or(false))
1382    }
1383
1384    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1385    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1386    pub async fn resolve_external_tool_call(
1387        &self,
1388        request_id: &str,
1389        resolution: ExternalToolResolution,
1390    ) -> anyhow::Result<bool> {
1391        let pending = self
1392            .pending_external_tool_calls
1393            .lock()
1394            .await
1395            .remove(request_id);
1396        let Some(pending) = pending else {
1397            return Ok(false);
1398        };
1399        self.emit(RoderEvent::ExternalToolCallResolved(
1400            ExternalToolCallResolved {
1401                thread_id: pending.thread_id,
1402                turn_id: pending.turn_id,
1403                request_id: request_id.to_string(),
1404                tool_id: pending.tool_id,
1405                tool_name: pending.tool_name,
1406                outcome: ExternalToolCallOutcome::Resolved,
1407                is_error: resolution.is_error,
1408                timestamp: OffsetDateTime::now_utc(),
1409            },
1410        ))
1411        .await;
1412        let _ = pending.tx.send(resolution);
1413        Ok(true)
1414    }
1415
1416    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1417    /// interrupt does not leak requests waiting on `tools/resolve`.
1418    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1419        let cancelled = {
1420            let mut pending = self.pending_external_tool_calls.lock().await;
1421            let request_ids = pending
1422                .iter()
1423                .filter(|(_, call)| &call.turn_id == turn_id)
1424                .map(|(request_id, _)| request_id.clone())
1425                .collect::<Vec<_>>();
1426            request_ids
1427                .into_iter()
1428                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1429                .collect::<Vec<_>>()
1430        };
1431        for (request_id, call) in cancelled {
1432            self.emit(RoderEvent::ExternalToolCallResolved(
1433                ExternalToolCallResolved {
1434                    thread_id: call.thread_id,
1435                    turn_id: call.turn_id,
1436                    request_id,
1437                    tool_id: call.tool_id,
1438                    tool_name: call.tool_name,
1439                    outcome: ExternalToolCallOutcome::Cancelled,
1440                    is_error: true,
1441                    timestamp: OffsetDateTime::now_utc(),
1442                },
1443            ))
1444            .await;
1445        }
1446    }
1447
1448    pub async fn resolve_user_input(
1449        &self,
1450        request_id: &str,
1451        answers: serde_json::Value,
1452    ) -> anyhow::Result<bool> {
1453        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1454        let Some(pending) = pending else {
1455            return Ok(false);
1456        };
1457        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1458            thread_id: pending.thread_id,
1459            turn_id: pending.turn_id,
1460            request_id: request_id.to_string(),
1461            answers: answers.clone(),
1462            timestamp: OffsetDateTime::now_utc(),
1463        }))
1464        .await;
1465        let _ = pending.tx.send(answers);
1466        Ok(true)
1467    }
1468
1469    async fn emit_plan_exit_resolved(
1470        &self,
1471        current: &PendingPlanExit,
1472        approved: bool,
1473        resolved_mode: PolicyMode,
1474    ) {
1475        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1476            thread_id: current.thread_id.clone(),
1477            turn_id: current.turn_id.clone(),
1478            request_id: current.request_id.clone(),
1479            approved,
1480            target_mode: current.target_mode,
1481            resolved_mode,
1482            timestamp: OffsetDateTime::now_utc(),
1483        }))
1484        .await;
1485    }
1486
1487    pub async fn select_provider(
1488        &self,
1489        provider: String,
1490        model: Option<String>,
1491        reasoning: Option<String>,
1492    ) -> anyhow::Result<RuntimeConfig> {
1493        let next = self
1494            .preview_provider_selection(provider, model, reasoning)
1495            .await?;
1496        let mut cfg = self.config.write().await;
1497        *cfg = next;
1498        Ok(cfg.clone())
1499    }
1500
1501    pub async fn preview_provider_selection(
1502        &self,
1503        provider: String,
1504        model: Option<String>,
1505        reasoning: Option<String>,
1506    ) -> anyhow::Result<RuntimeConfig> {
1507        self.engine_for(&provider)?;
1508        let mut cfg = self.config.read().await.clone();
1509        cfg.default_provider = provider;
1510        if let Some(model) = model {
1511            cfg.default_model = model;
1512        }
1513        if let Some(reasoning) = reasoning {
1514            if reasoning == REASONING_NONE
1515                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1516            {
1517                return Ok(cfg.clone());
1518            }
1519            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1520            cfg.reasoning = Some(reasoning);
1521        }
1522        Ok(cfg)
1523    }
1524
1525    pub async fn effective_reasoning(&self) -> String {
1526        let cfg = self.config.read().await;
1527        effective_reasoning_for_model(&cfg, &cfg.default_model)
1528    }
1529
1530    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1531        effective_reasoning_for_model(cfg, &cfg.default_model)
1532    }
1533
1534    pub async fn set_dynamic_workflow_effort(
1535        &self,
1536        effort_profile: DynamicWorkflowEffortProfile,
1537    ) -> RuntimeConfig {
1538        let mut cfg = self.config.write().await;
1539        cfg.dynamic_workflows.effort_profile = effort_profile;
1540        cfg.clone()
1541    }
1542
1543    pub async fn dynamic_workflow_trigger_decision(
1544        &self,
1545        message: &str,
1546    ) -> WorkflowTriggerDecision {
1547        let cfg = self.config.read().await;
1548        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1549    }
1550
1551    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1552        self.create_thread_with(CreateThreadRequest {
1553            title,
1554            workspace: self.workspace.display().to_string(),
1555            workspace_id: None,
1556            root_id: None,
1557            provider: None,
1558            model: None,
1559            selection_mode: None,
1560            tool_allowlist: Vec::new(),
1561            developer_instructions: None,
1562            external_tools: Vec::new(),
1563            runner: None,
1564        })
1565        .await
1566    }
1567
1568    /**
1569     * Resolves an explicit thread-runner selection into a persisted binding.
1570     * Fails fast at thread creation when the provider is missing or rejects
1571     * the destination, instead of surfacing the error on the first tool call.
1572     */
1573    async fn resolve_thread_runner_binding(
1574        &self,
1575        thread_id: &str,
1576        selection: ThreadRunnerSelection,
1577    ) -> anyhow::Result<ThreadRunnerBinding> {
1578        let provider = self
1579            .registry
1580            .remote_runner_providers
1581            .iter()
1582            .find(|provider| provider.id() == selection.provider_id)
1583            .cloned()
1584            .ok_or_else(|| {
1585                anyhow::anyhow!(
1586                    "remote runner provider {:?} is not installed",
1587                    selection.provider_id
1588                )
1589            })?;
1590        let workspace = selection.workspace.trim();
1591        anyhow::ensure!(
1592            std::path::Path::new(workspace).is_absolute(),
1593            "runner workspace must be an absolute path on the runner: {workspace:?}"
1594        );
1595        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1596        for read_root in &selection.read_roots {
1597            let trimmed = read_root.trim();
1598            anyhow::ensure!(
1599                std::path::Path::new(trimmed).is_absolute(),
1600                "runner read root must be an absolute path on the runner: {trimmed:?}"
1601            );
1602            read_roots.push(PathBuf::from(trimmed));
1603        }
1604        let destination = RunnerDestination {
1605            id: format!("thread-{thread_id}"),
1606            provider_id: selection.provider_id,
1607            config: selection.config,
1608            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1609        };
1610        provider.validate_destination(&destination).await?;
1611        Ok(ThreadRunnerBinding {
1612            destination,
1613            workspace: PathBuf::from(workspace),
1614            read_roots,
1615        })
1616    }
1617
1618    /**
1619     * Build a per-thread binding from a runtime-level destination (selected via
1620     * the TUI runner picker or config `default_destination`) when the
1621     * destination's provider advertises a default workspace. Returns `None` when
1622     * there is no destination or the provider opts out (no default workspace),
1623     * preserving the legacy behavior where such threads keep local tools.
1624     */
1625    async fn synthesize_runtime_runner_binding(
1626        &self,
1627        thread_id: &str,
1628        destination: Option<RunnerDestination>,
1629    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1630        let Some(destination) = destination else {
1631            return Ok(None);
1632        };
1633        let Some(provider) = self
1634            .registry
1635            .remote_runner_providers
1636            .iter()
1637            .find(|provider| provider.id() == destination.provider_id)
1638        else {
1639            return Ok(None);
1640        };
1641        // An explicit workspace in the destination config wins over the
1642        // provider default; absence of both means the provider opts out.
1643        let workspace = destination
1644            .config
1645            .get("working_dir")
1646            .and_then(serde_json::Value::as_str)
1647            .map(str::to_string)
1648            .or_else(|| provider.default_workspace());
1649        let Some(workspace) = workspace else {
1650            return Ok(None);
1651        };
1652        let selection = ThreadRunnerSelection {
1653            provider_id: destination.provider_id.clone(),
1654            config: destination.config.clone(),
1655            workspace,
1656            read_roots: Vec::new(),
1657        };
1658        Ok(Some(
1659            self.resolve_thread_runner_binding(thread_id, selection)
1660                .await?,
1661        ))
1662    }
1663
1664    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1665    pub async fn validate_thread_runner_selection(
1666        &self,
1667        selection: ThreadRunnerSelection,
1668    ) -> anyhow::Result<()> {
1669        self.resolve_thread_runner_binding("validate", selection)
1670            .await
1671            .map(|_| ())
1672    }
1673
1674    pub async fn create_thread_with(
1675        &self,
1676        req: CreateThreadRequest,
1677    ) -> anyhow::Result<ThreadMetadata> {
1678        let cfg = self.config.read().await.clone();
1679        let now = OffsetDateTime::now_utc();
1680        let workspace = validate_thread_workspace(&req.workspace)?;
1681        let provider = req.provider.unwrap_or(cfg.default_provider);
1682        let model = req.model.unwrap_or(cfg.default_model);
1683        let selection_mode = req
1684            .selection_mode
1685            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1686        let thread_id = uuid::Uuid::new_v4().to_string();
1687        let runner_binding = match req.runner {
1688            Some(selection) => Some(
1689                self.resolve_thread_runner_binding(&thread_id, selection)
1690                    .await?,
1691            ),
1692            // No explicit per-thread selection: if a runtime-level destination
1693            // is active and its provider advertises a default workspace, bind
1694            // the new thread so its coding tools route into the runner. This is
1695            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1696            // execute tools in the sandbox instead of locally. Providers that
1697            // return no default workspace stay unbound (legacy local-tools).
1698            None => {
1699                self.synthesize_runtime_runner_binding(
1700                    &thread_id,
1701                    cfg.remote_runner_destination.clone(),
1702                )
1703                .await?
1704            }
1705        };
1706        let runner_destination = runner_binding
1707            .as_ref()
1708            .map(|binding| binding.destination.clone())
1709            .or_else(|| cfg.remote_runner_destination.clone());
1710        let metadata = ThreadMetadata {
1711            thread_id,
1712            title: req.title,
1713            workspace,
1714            workspace_id: req.workspace_id,
1715            root_id: req.root_id,
1716            provider: Some(provider),
1717            model: Some(model),
1718            selection_mode: Some(selection_mode),
1719            tool_allowlist: req.tool_allowlist,
1720            developer_instructions: req.developer_instructions,
1721            external_tools: req.external_tools,
1722            runner_destination,
1723            runner_state: None,
1724            runner_binding,
1725            created_at: now,
1726            updated_at: now,
1727            message_count: 0,
1728            usage: None,
1729            parent_thread_id: None,
1730            forked_from_turn_id: None,
1731            workspace_fork: None,
1732        };
1733
1734        let metadata = if let Some(store) = &self.thread_store {
1735            store.create_thread(metadata).await?
1736        } else {
1737            metadata
1738        };
1739        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1740            thread_id: metadata.thread_id.clone(),
1741            timestamp: OffsetDateTime::now_utc(),
1742        }))
1743        .await;
1744        Ok(metadata)
1745    }
1746
1747    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1748        if let Some(store) = &self.thread_store {
1749            return store.list_threads().await;
1750        }
1751        Ok(Vec::new())
1752    }
1753
1754    pub async fn list_threads_page(
1755        &self,
1756        options: roder_api::thread::ThreadListOptions,
1757    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1758        if let Some(store) = &self.thread_store {
1759            return store.list_threads_page(options).await;
1760        }
1761        Ok(roder_api::thread::ThreadListPage::default())
1762    }
1763
1764    pub async fn load_thread_metadata(
1765        &self,
1766        thread_id: &str,
1767    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1768        if let Some(store) = &self.thread_store {
1769            return store.load_thread_metadata(&thread_id.to_string()).await;
1770        }
1771        Ok(None)
1772    }
1773
1774    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1775        let archived = if let Some(store) = &self.thread_store {
1776            store.archive_thread(&thread_id.to_string()).await?
1777        } else {
1778            false
1779        };
1780        if archived {
1781            self.thread_item_cache
1782                .lock()
1783                .await
1784                .remove_thread(&thread_id.to_string());
1785            self.evict_runner_session(&thread_id.to_string()).await;
1786        }
1787        Ok(archived)
1788    }
1789
1790    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1791        let cfg = self.config.read().await.clone();
1792        let workspace = self.workspace.display().to_string();
1793        let lead_thread_id = match req.lead_thread_id {
1794            Some(thread_id) => thread_id,
1795            None => {
1796                self.create_thread_with(CreateThreadRequest {
1797                    title: Some("Team lead".to_string()),
1798                    workspace: workspace.clone(),
1799                    workspace_id: None,
1800                    root_id: None,
1801                    provider: None,
1802                    model: None,
1803                    selection_mode: None,
1804                    tool_allowlist: Vec::new(),
1805                    developer_instructions: None,
1806                    external_tools: Vec::new(),
1807                    runner: None,
1808                })
1809                .await?
1810                .thread_id
1811            }
1812        };
1813        let team_id = uuid::Uuid::new_v4().to_string();
1814        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1815        let lead_selection = match active_lead_turn_id.as_ref() {
1816            Some(turn_id) => self
1817                .active_turn_selections
1818                .read()
1819                .await
1820                .get(turn_id)
1821                .cloned(),
1822            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1823        };
1824        let lead_concrete_selection = lead_selection
1825            .as_ref()
1826            .map(ModelSelectionMode::concrete_selection);
1827        let mut lead = crate::teams::lead_member(
1828            lead_thread_id.clone(),
1829            lead_concrete_selection
1830                .as_ref()
1831                .map(|selection| selection.provider.clone())
1832                .or_else(|| Some(cfg.default_provider.clone())),
1833            lead_concrete_selection
1834                .as_ref()
1835                .map(|selection| selection.model.clone())
1836                .or_else(|| Some(cfg.default_model.clone())),
1837            cfg.policy_mode,
1838        );
1839        if let Some(turn_id) = active_lead_turn_id {
1840            lead.current_turn_id = Some(turn_id);
1841            lead.status = TeamMemberStatus::Running;
1842        }
1843        let mut members = vec![lead];
1844
1845        for (index, member) in req.members.into_iter().enumerate() {
1846            let thread = self
1847                .create_thread_with(CreateThreadRequest {
1848                    title: Some(member.name.clone()),
1849                    workspace: workspace.clone(),
1850                    workspace_id: None,
1851                    root_id: None,
1852                    provider: member.model_provider.clone(),
1853                    model: member.model.clone(),
1854                    selection_mode: None,
1855                    tool_allowlist: Vec::new(),
1856                    developer_instructions: None,
1857                    external_tools: Vec::new(),
1858                    runner: None,
1859                })
1860                .await?;
1861            let member_id = format!("member-{}", index + 1);
1862            let descriptor = crate::teams::teammate_member(
1863                member_id.clone(),
1864                member.name,
1865                thread.thread_id.clone(),
1866                member.model_provider.or(thread.provider),
1867                member.model.or(thread.model),
1868                cfg.policy_mode,
1869            );
1870            members.push(descriptor);
1871        }
1872
1873        let now = OffsetDateTime::now_utc();
1874        let team = self
1875            .teams
1876            .insert(TeamState {
1877                id: team_id.clone(),
1878                lead_thread_id: lead_thread_id.clone(),
1879                display_mode: req.display_mode,
1880                members,
1881                mailbox: Vec::new(),
1882                tasks: Vec::new(),
1883                created_at: now,
1884                updated_at: now,
1885            })
1886            .await?;
1887        self.emit(RoderEvent::TeamStarted(TeamStarted {
1888            team_id: team_id.clone(),
1889            lead_thread_id,
1890            display_mode: team.display_mode,
1891            members: team.members.clone(),
1892            tasks: team.tasks.clone(),
1893            timestamp: OffsetDateTime::now_utc(),
1894        }))
1895        .await;
1896        for member in team
1897            .members
1898            .iter()
1899            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1900        {
1901            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1902                team_id: team_id.clone(),
1903                member: member.clone(),
1904                timestamp: OffsetDateTime::now_utc(),
1905            }))
1906            .await;
1907        }
1908        Ok(team)
1909    }
1910
1911    pub async fn list_teams(&self) -> Vec<TeamState> {
1912        let active_thread_ids = self
1913            .active_turns
1914            .read()
1915            .await
1916            .values()
1917            .map(|handle| handle.thread_id.clone())
1918            .collect::<std::collections::HashSet<_>>();
1919        self.teams
1920            .list()
1921            .await
1922            .into_iter()
1923            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1924            .collect()
1925    }
1926
1927    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1928        let active_thread_ids = self
1929            .active_turns
1930            .read()
1931            .await
1932            .values()
1933            .map(|handle| handle.thread_id.clone())
1934            .collect::<std::collections::HashSet<_>>();
1935        self.teams
1936            .get(team_id)
1937            .await
1938            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1939    }
1940
1941    pub async fn start_team_member(
1942        &self,
1943        team_id: &str,
1944        req: TeamMemberStartRequest,
1945    ) -> anyhow::Result<TeamState> {
1946        self.start_team_member_with_selection(team_id, req, None)
1947            .await
1948    }
1949
1950    pub async fn message_team_member(
1951        self: &Arc<Self>,
1952        team_id: &str,
1953        member_id: &str,
1954        message: String,
1955    ) -> anyhow::Result<TurnId> {
1956        let team = self
1957            .read_team(team_id)
1958            .await
1959            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1960        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
1961            .await
1962    }
1963
1964    /// Queue a mailbox message without starting an idle agent. If the target is
1965    /// already running, the message is delivered at the next inference boundary.
1966    pub(crate) async fn queue_team_member_message(
1967        self: &Arc<Self>,
1968        caller_thread_id: &ThreadId,
1969        team_id: &str,
1970        member_id: &str,
1971        message: String,
1972    ) -> anyhow::Result<Option<TurnId>> {
1973        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
1974        let team = self
1975            .read_team(team_id)
1976            .await
1977            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1978        let member = team
1979            .members
1980            .iter()
1981            .find(|member| member.id == member_id)
1982            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
1983            .clone();
1984        if member.status == TeamMemberStatus::Closed {
1985            anyhow::bail!("subagent {} is closed", member.name);
1986        }
1987        let from_member_id = team
1988            .members
1989            .iter()
1990            .find(|candidate| candidate.thread_id == *caller_thread_id)
1991            .map(|candidate| candidate.id.clone());
1992        self.teams
1993            .append_mailbox_message(
1994                team_id,
1995                from_member_id,
1996                member_id.to_string(),
1997                TeamMailboxMessageKind::Message,
1998                message,
1999            )
2000            .await?;
2001        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2002            return Ok(None);
2003        };
2004        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2005            .await?;
2006        Ok(Some(turn_id))
2007    }
2008
2009    /// Deliver queued messages and start an idle agent, or steer its active turn.
2010    pub(crate) async fn followup_team_member(
2011        self: &Arc<Self>,
2012        caller_thread_id: &ThreadId,
2013        team_id: &str,
2014        member_id: &str,
2015        message: String,
2016    ) -> anyhow::Result<TurnId> {
2017        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2018        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2019            .await
2020    }
2021
2022    async fn followup_team_member_locked(
2023        self: &Arc<Self>,
2024        caller_thread_id: &ThreadId,
2025        team_id: &str,
2026        member_id: &str,
2027        message: String,
2028    ) -> anyhow::Result<TurnId> {
2029        let team = self
2030            .read_team(team_id)
2031            .await
2032            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2033        let member = team
2034            .members
2035            .iter()
2036            .find(|member| member.id == member_id)
2037            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2038            .clone();
2039        if member.status == TeamMemberStatus::Closed {
2040            anyhow::bail!("subagent {} is closed", member.name);
2041        }
2042        let from_member_id = team
2043            .members
2044            .iter()
2045            .find(|candidate| candidate.thread_id == *caller_thread_id)
2046            .map(|candidate| candidate.id.clone());
2047        self.teams
2048            .append_mailbox_message(
2049                team_id,
2050                from_member_id,
2051                member_id.to_string(),
2052                TeamMailboxMessageKind::NewTask,
2053                message,
2054            )
2055            .await?;
2056        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2057            self.deliver_pending_team_mailbox(
2058                team_id,
2059                member_id,
2060                member.thread_id,
2061                turn_id.clone(),
2062            )
2063            .await?;
2064            return Ok(turn_id);
2065        }
2066
2067        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2068            .await?;
2069        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2070        let inherited_context = self
2071            .team_member_turn_contexts
2072            .lock()
2073            .await
2074            .get(&member.thread_id)
2075            .cloned();
2076        let workspace = inherited_context
2077            .as_ref()
2078            .map(|context| context.workspace.clone())
2079            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2080            .unwrap_or_else(|| self.workspace.display().to_string());
2081        let member_thread_id = member.thread_id;
2082        let turn_id = self
2083            .start_turn(StartTurnRequest {
2084                thread_id: member_thread_id.clone(),
2085                message: String::new(),
2086                images: Vec::new(),
2087                provider_override: member.model_provider,
2088                model_override: member.model,
2089                reasoning_override: metadata
2090                    .as_ref()
2091                    .and_then(|metadata| metadata.selection_mode.as_ref())
2092                    .and_then(ModelSelectionMode::reasoning)
2093                    .map(str::to_string),
2094                workspace,
2095                instructions: inherited_context
2096                    .as_ref()
2097                    .map(|context| context.instructions.clone())
2098                    .unwrap_or_else(crate::default_instructions),
2099                developer_context: inherited_context
2100                    .as_ref()
2101                    .and_then(|context| context.developer_context.clone()),
2102                task_ledger_required: false,
2103            })
2104            .await?;
2105        Ok(turn_id)
2106    }
2107
2108    pub async fn set_team_member_policy_mode(
2109        &self,
2110        team_id: &str,
2111        member_id: &str,
2112        policy_mode: PolicyMode,
2113    ) -> anyhow::Result<TeamState> {
2114        self.teams
2115            .set_member_policy_mode(team_id, member_id, policy_mode)
2116            .await
2117    }
2118
2119    pub async fn interrupt_team_member(
2120        &self,
2121        team_id: &str,
2122        member_id: &str,
2123    ) -> anyhow::Result<Option<TurnId>> {
2124        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2125        let team = self
2126            .read_team(team_id)
2127            .await
2128            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2129        let member = team
2130            .members
2131            .iter()
2132            .find(|member| member.id == member_id)
2133            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2134            .clone();
2135        if member.status != TeamMemberStatus::Running {
2136            return Ok(None);
2137        }
2138        let Some(turn_id) = member.current_turn_id.clone() else {
2139            return Ok(None);
2140        };
2141        if self
2142            .active_turn_for_thread(&member.thread_id)
2143            .await
2144            .as_ref()
2145            != Some(&turn_id)
2146        {
2147            return Ok(None);
2148        }
2149        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2150            .await?;
2151        // Make queued mailbox work immediately eligible for the replacement turn while the
2152        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2153        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2154        // reservation acquired by its replacement.
2155        self.teams
2156            .release_mailbox_reservations_for_turn(&turn_id)
2157            .await;
2158        self.teams
2159            .update_member(team_id, member_id, |member| {
2160                member.status = TeamMemberStatus::Interrupted;
2161                member.current_turn_id = None;
2162                member.terminal_error = None;
2163            })
2164            .await?;
2165        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2166            team_id: team_id.to_string(),
2167            member_id: member_id.to_string(),
2168            member_thread_id: member.thread_id,
2169            turn_id: Some(turn_id.clone()),
2170            status: TeamMemberStatus::Interrupted,
2171            final_message: member.final_message,
2172            error: None,
2173            timestamp: OffsetDateTime::now_utc(),
2174        }))
2175        .await;
2176        Ok(Some(turn_id))
2177    }
2178
2179    pub async fn close_team_member(
2180        &self,
2181        team_id: &str,
2182        member_id: &str,
2183    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2184        let team = self
2185            .read_team(team_id)
2186            .await
2187            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2188        let member = team
2189            .members
2190            .iter()
2191            .find(|member| member.id == member_id)
2192            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2193            .clone();
2194        if member.role == roder_api::teams::TeamMemberRole::Lead {
2195            anyhow::bail!("team lead cannot be closed as a subagent");
2196        }
2197        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2198            if let Some(turn_id) = member.current_turn_id.clone() {
2199                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2200                    .await?;
2201                Some(turn_id)
2202            } else {
2203                None
2204            }
2205        } else {
2206            member.current_turn_id.clone()
2207        };
2208        let updated = self
2209            .teams
2210            .update_member(team_id, member_id, |member| {
2211                member.status = TeamMemberStatus::Closed;
2212                member.current_turn_id = None;
2213            })
2214            .await?;
2215        let closed = updated
2216            .members
2217            .iter()
2218            .find(|member| member.id == member_id)
2219            .cloned()
2220            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2221        self.team_member_turn_contexts
2222            .lock()
2223            .await
2224            .remove(&closed.thread_id);
2225        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2226            team_id: team_id.to_string(),
2227            member_id: closed.id.clone(),
2228            member_thread_id: closed.thread_id.clone(),
2229            turn_id: interrupted_turn_id,
2230            status: TeamMemberStatus::Closed,
2231            final_message: closed.final_message.clone(),
2232            error: closed.terminal_error.clone(),
2233            timestamp: OffsetDateTime::now_utc(),
2234        }))
2235        .await;
2236        Ok(closed)
2237    }
2238
2239    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2240        let Some(team) = self.read_team(team_id).await else {
2241            return Ok(false);
2242        };
2243        if !force
2244            && team
2245                .members
2246                .iter()
2247                .any(|member| member.status == TeamMemberStatus::Running)
2248        {
2249            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2250        }
2251        if force {
2252            for member in team.members.iter().filter(|member| {
2253                member.role != roder_api::teams::TeamMemberRole::Lead
2254                    && member.status == TeamMemberStatus::Running
2255            }) {
2256                if let Some(turn_id) = member
2257                    .current_turn_id
2258                    .clone()
2259                    .or(self.active_turn_for_thread(&member.thread_id).await)
2260                {
2261                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2262                }
2263            }
2264        }
2265        let removed = self.teams.remove(team_id).await?.is_some();
2266        if removed {
2267            let member_thread_ids = team
2268                .members
2269                .iter()
2270                .map(|member| member.thread_id.clone())
2271                .collect::<std::collections::HashSet<_>>();
2272            self.team_member_turn_contexts
2273                .lock()
2274                .await
2275                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2276            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2277                team_id: team_id.to_string(),
2278                forced: force,
2279                timestamp: OffsetDateTime::now_utc(),
2280            }))
2281            .await;
2282        }
2283        Ok(removed)
2284    }
2285
2286    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2287        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2288            return mode;
2289        }
2290        self.status().await.policy_mode
2291    }
2292
2293    async fn complete_team_member_turn_with_result(
2294        &self,
2295        thread_id: &ThreadId,
2296        turn_id: &TurnId,
2297        status: TeamMemberStatus,
2298        final_message: Option<String>,
2299        terminal_error: Option<String>,
2300    ) -> anyhow::Result<()> {
2301        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2302        let Some((team_id, member)) = self
2303            .teams
2304            .complete_member_turn(
2305                thread_id,
2306                turn_id,
2307                status,
2308                final_message.clone(),
2309                terminal_error.clone(),
2310            )
2311            .await?
2312        else {
2313            return Ok(());
2314        };
2315        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2316            && let Some(team) = self.read_team(&team_id).await
2317            && let Some(parent) = team
2318                .members
2319                .iter()
2320                .find(|candidate| candidate.thread_id == *parent_thread_id)
2321        {
2322            let identity = member
2323                .agent_path
2324                .as_deref()
2325                .or(member.task_name.as_deref())
2326                .unwrap_or(&member.name);
2327            let mut report = format!("Agent {identity} finished with status {status:?}.");
2328            if let Some(message) = final_message.as_deref()
2329                && !message.trim().is_empty()
2330            {
2331                report.push_str("\n\nFinal result:\n");
2332                report.push_str(message);
2333            }
2334            if let Some(error) = terminal_error.as_deref()
2335                && !error.trim().is_empty()
2336            {
2337                report.push_str("\n\nTerminal error:\n");
2338                report.push_str(error);
2339            }
2340            let mailbox_appended = self
2341                .teams
2342                .append_mailbox_message(
2343                    &team_id,
2344                    Some(member.id.clone()),
2345                    parent.id.clone(),
2346                    TeamMailboxMessageKind::FinalAnswer,
2347                    report,
2348                )
2349                .await
2350                .is_ok();
2351            if mailbox_appended
2352                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2353            {
2354                // The parent may finish between the active-turn lookup and enqueue. Its durable
2355                // mailbox entry remains pending for the next turn, while terminal observers must
2356                // still receive TeamMemberCompleted for this child.
2357                let _ = self
2358                    .deliver_pending_team_mailbox(
2359                        &team_id,
2360                        &parent.id,
2361                        parent_thread_id.clone(),
2362                        parent_turn_id,
2363                    )
2364                    .await;
2365            }
2366        }
2367        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2368            team_id,
2369            member_id: member.id,
2370            member_thread_id: member.thread_id,
2371            turn_id: Some(turn_id.clone()),
2372            status,
2373            final_message,
2374            error: terminal_error,
2375            timestamp: OffsetDateTime::now_utc(),
2376        }))
2377        .await;
2378        Ok(())
2379    }
2380
2381    /// Stops accepting new turns, requests interruption for every active turn,
2382    /// and waits for their runtime tasks to reach terminal cleanup up to
2383    /// `timeout`. Repeated calls are safe and continue draining the same set.
2384    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2385        let started_at = tokio::time::Instant::now();
2386        let persistence_failures_before =
2387            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2388        let turn_admission = self.turn_admission.lock().await;
2389        self.accepting_turns.store(false, Ordering::Release);
2390        let active = self
2391            .active_turns
2392            .read()
2393            .await
2394            .iter()
2395            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2396            .collect::<Vec<_>>();
2397        let draining = self
2398            .turn_drains
2399            .read()
2400            .await
2401            .iter()
2402            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2403            .collect::<Vec<_>>();
2404        drop(turn_admission);
2405
2406        let mut interrupted_turns = std::collections::BTreeMap::new();
2407        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2408            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2409        }
2410        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2411        for (turn_id, thread_id) in active {
2412            let _ = self
2413                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2414                .await;
2415        }
2416
2417        let wait_for_empty = async {
2418            loop {
2419                let notified = self.active_turns_changed.notified();
2420                if self.active_turns.read().await.is_empty()
2421                    && self.turn_drains.read().await.is_empty()
2422                {
2423                    return;
2424                }
2425                notified.await;
2426            }
2427        };
2428        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2429            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2430                > persistence_failures_before
2431            {
2432                RuntimeDrainOutcome::PersistenceFailed {
2433                    interrupted_turn_ids,
2434                    remaining_turn_ids: Vec::new(),
2435                }
2436            } else {
2437                RuntimeDrainOutcome::Clean {
2438                    interrupted_turn_ids,
2439                }
2440            }
2441        } else {
2442            let active_remaining = self
2443                .active_turns
2444                .read()
2445                .await
2446                .iter()
2447                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2448                .collect::<Vec<_>>();
2449            let draining_remaining = self
2450                .turn_drains
2451                .read()
2452                .await
2453                .iter()
2454                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2455                .collect::<Vec<_>>();
2456            let remaining = active_remaining
2457                .into_iter()
2458                .chain(draining_remaining)
2459                .collect::<std::collections::BTreeMap<_, _>>();
2460            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2461            for turn_id in &remaining_turn_ids {
2462                let handle = remaining
2463                    .get(turn_id)
2464                    .expect("remaining turn ID must have a drain handle");
2465                self.record_turn_lifecycle(
2466                    handle.thread_id.clone(),
2467                    turn_id.clone(),
2468                    TurnLifecycleState::InterruptRequested,
2469                    TurnCleanupState::TimedOut,
2470                    Some(TurnLifecycleReason::Shutdown),
2471                )
2472                .await;
2473            }
2474            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2475                > persistence_failures_before
2476            {
2477                RuntimeDrainOutcome::PersistenceFailed {
2478                    interrupted_turn_ids,
2479                    remaining_turn_ids,
2480                }
2481            } else {
2482                RuntimeDrainOutcome::DeadlineExceeded {
2483                    interrupted_turn_ids,
2484                    remaining_turn_ids,
2485                }
2486            }
2487        };
2488        self.record_lifecycle_drain_outcome(started_at, &outcome);
2489        outcome
2490    }
2491
2492    /// Enables turn submission after a prior drain attempt. This is intended
2493    /// for embedders that keep a runtime alive after a bounded drain timeout.
2494    pub fn resume_accepting_turns(&self) {
2495        self.accepting_turns.store(true, Ordering::Release);
2496    }
2497
2498    pub async fn turn_lifecycle_snapshot(
2499        &self,
2500        thread_id: &ThreadId,
2501    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2502        let Some(store) = &self.thread_store else {
2503            return Ok(TurnLifecycleSnapshot::default());
2504        };
2505        let extension_states = store.load_extension_states(thread_id).await?;
2506        Ok(turn_lifecycle_snapshot(&extension_states))
2507    }
2508
2509    pub async fn load_thread(
2510        &self,
2511        thread_id: &ThreadId,
2512    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2513        let loaded = if let Some(store) = &self.thread_store {
2514            store.load_thread(thread_id).await?
2515        } else {
2516            None
2517        };
2518        if let Some(snapshot) = loaded.as_ref() {
2519            let live_turn_ids = self
2520                .active_turns
2521                .read()
2522                .await
2523                .keys()
2524                .cloned()
2525                .chain(self.turn_drains.read().await.keys().cloned())
2526                .collect::<std::collections::HashSet<_>>();
2527            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2528            for record in lifecycle.records.into_iter().filter(|record| {
2529                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2530            }) {
2531                self.lifecycle_restart_reconciliations
2532                    .fetch_add(1, Ordering::AcqRel);
2533                self.record_turn_lifecycle(
2534                    record.thread_id,
2535                    record.turn_id,
2536                    TurnLifecycleState::RecoveryNeeded,
2537                    TurnCleanupState::Unknown,
2538                    Some(TurnLifecycleReason::RuntimeRestart),
2539                )
2540                .await;
2541            }
2542            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2543                thread_id: thread_id.clone(),
2544                timestamp: OffsetDateTime::now_utc(),
2545            }))
2546            .await;
2547        }
2548        Ok(loaded)
2549    }
2550
2551    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2552        if let Some(store) = &self.thread_store {
2553            let snapshot = store
2554                .load_thread(thread_id)
2555                .await?
2556                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2557            match snapshot {
2558                Ok(snapshot) => {
2559                    if let Some(metadata) = snapshot.metadata {
2560                        // Fork-backed threads fail closed before any write
2561                        // when their workspace was removed out-of-band.
2562                        if let Some(fork) = &metadata.workspace_fork
2563                            && fork.status == roder_api::forks::ForkStatus::Active
2564                            && !std::path::Path::new(&metadata.workspace).is_dir()
2565                        {
2566                            anyhow::bail!(
2567                                "workspace fork {} is missing its workspace at {}; restore it or \
2568                                 remove the fork before running turns in this thread",
2569                                fork.id,
2570                                metadata.workspace
2571                            );
2572                        }
2573                        return Ok(metadata.workspace);
2574                    }
2575                    eprintln!(
2576                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2577                    );
2578                }
2579                Err(err) => {
2580                    eprintln!(
2581                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2582                    );
2583                }
2584            }
2585        }
2586        Ok(self.workspace.display().to_string())
2587    }
2588
2589    async fn selection_mode_for_thread(
2590        &self,
2591        thread_id: &ThreadId,
2592    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2593        let Some(store) = &self.thread_store else {
2594            return Ok(None);
2595        };
2596        Ok(store
2597            .load_thread(thread_id)
2598            .await?
2599            .and_then(|snapshot| snapshot.metadata)
2600            .and_then(|metadata| {
2601                metadata
2602                    .selection_mode
2603                    .or_else(|| match (metadata.provider, metadata.model) {
2604                        (Some(provider), Some(model)) => {
2605                            Some(ModelSelectionMode::manual(provider, model, None))
2606                        }
2607                        _ => None,
2608                    })
2609            }))
2610    }
2611
2612    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2613    pub(crate) async fn thread_turn_overrides(
2614        &self,
2615        thread_id: &ThreadId,
2616    ) -> anyhow::Result<ThreadTurnOverrides> {
2617        let Some(store) = &self.thread_store else {
2618            return Ok(ThreadTurnOverrides::default());
2619        };
2620        Ok(store
2621            .load_thread_metadata(thread_id)
2622            .await?
2623            .map(|metadata| ThreadTurnOverrides {
2624                tool_allowlist: metadata.tool_allowlist,
2625                developer_instructions: metadata.developer_instructions,
2626                external_tools: metadata.external_tools,
2627            })
2628            .unwrap_or_default())
2629    }
2630
2631    pub async fn set_thread_selection_mode(
2632        &self,
2633        thread_id: &ThreadId,
2634        selection_mode: ModelSelectionMode,
2635    ) -> anyhow::Result<()> {
2636        let Some(store) = &self.thread_store else {
2637            return Ok(());
2638        };
2639        let Some(snapshot) = store.load_thread(thread_id).await? else {
2640            anyhow::bail!("thread not found: {thread_id}");
2641        };
2642        let Some(mut metadata) = snapshot.metadata else {
2643            return Ok(());
2644        };
2645        let concrete = selection_mode.concrete_selection();
2646        metadata.provider = Some(concrete.provider);
2647        metadata.model = Some(concrete.model);
2648        metadata.selection_mode = Some(selection_mode);
2649        metadata.updated_at = OffsetDateTime::now_utc();
2650        store.update_thread_metadata(metadata).await?;
2651        Ok(())
2652    }
2653
2654    async fn runner_session_for_thread(
2655        &self,
2656        thread_id: &ThreadId,
2657    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2658        let metadata = if let Some(store) = &self.thread_store {
2659            store.load_thread_metadata(thread_id).await?
2660        } else {
2661            None
2662        };
2663        // An explicit per-thread binding wins over the runtime-level destination.
2664        let destination = metadata
2665            .as_ref()
2666            .and_then(|metadata| metadata.runner_binding.as_ref())
2667            .map(|binding| binding.destination.clone())
2668            .or(self.config.read().await.remote_runner_destination.clone());
2669        let Some(destination) = destination else {
2670            self.evict_runner_session(thread_id).await;
2671            return Ok(None);
2672        };
2673        let provider = self
2674            .registry
2675            .remote_runner_providers
2676            .iter()
2677            .find(|provider| provider.id() == destination.provider_id)
2678            .cloned()
2679            .ok_or_else(|| {
2680                anyhow::anyhow!(
2681                    "remote runner provider {:?} is not installed",
2682                    destination.provider_id
2683                )
2684            })?;
2685        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2686        let slot = {
2687            let mut sessions = self.runner_sessions.lock().await;
2688            match sessions.get(thread_id) {
2689                Some(slot) if slot.matches(&destination) => slot.clone(),
2690                _ => {
2691                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2692                    sessions.insert(thread_id.clone(), slot.clone());
2693                    slot
2694                }
2695            }
2696        };
2697
2698        loop {
2699            let initialized = slot.initialized.notified();
2700            let mut state = slot.state.lock().await;
2701            match &*state {
2702                RunnerSessionSlotState::Ready(cached) => {
2703                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2704                }
2705                RunnerSessionSlotState::Failed(error) => {
2706                    return Err(anyhow::anyhow!(error.to_string()));
2707                }
2708                RunnerSessionSlotState::Initializing => {
2709                    drop(state);
2710                    initialized.await;
2711                }
2712                RunnerSessionSlotState::Empty => {
2713                    *state = RunnerSessionSlotState::Initializing;
2714                    drop(state);
2715                    self.spawn_runner_session_initialization(
2716                        thread_id.clone(),
2717                        destination.clone(),
2718                        provider.clone(),
2719                        persisted_state.clone(),
2720                        slot.clone(),
2721                    );
2722                }
2723            }
2724        }
2725    }
2726
2727    fn spawn_runner_session_initialization(
2728        &self,
2729        thread_id: ThreadId,
2730        destination: RunnerDestination,
2731        provider: Arc<dyn RemoteRunnerProvider>,
2732        persisted_state: Option<RunnerSessionState>,
2733        slot: Arc<RunnerSessionSlot>,
2734    ) {
2735        let thread_store = self.thread_store.clone();
2736        let runner_sessions = self.runner_sessions.clone();
2737        // The task intentionally outlives the turn that first requested the
2738        // workspace. Interrupting that turn must not strand the slot in
2739        // `Initializing` and deadlock every later tool or lifecycle call.
2740        drop(tokio::spawn(async move {
2741            let initialized = async {
2742                let session = if let Some(state) = persisted_state
2743                    && state.provider_id == destination.provider_id
2744                    && state.destination_id == destination.id
2745                {
2746                    match provider.resume_session(state).await {
2747                        Ok(session) => session,
2748                        Err(_) => provider.create_session(destination.clone()).await?,
2749                    }
2750                } else {
2751                    provider.create_session(destination.clone()).await?
2752                };
2753                let resolved = (destination.clone(), session.clone());
2754                // Persist before releasing the singleflight or dispatching
2755                // the first tool. A later process can rejoin even if the turn
2756                // that requested initialization has already been interrupted.
2757                Self::persist_runner_state_in_store(
2758                    thread_store.as_ref(),
2759                    &thread_id,
2760                    Some(&resolved),
2761                )
2762                .await?;
2763                Ok::<_, anyhow::Error>(CachedRunnerSession {
2764                    destination,
2765                    session,
2766                })
2767            }
2768            .await;
2769
2770            match initialized {
2771                Ok(cached) => {
2772                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2773                    slot.initialized.notify_waiters();
2774                }
2775                Err(error) => {
2776                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2777                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2778                    slot.initialized.notify_waiters();
2779                    let mut sessions = runner_sessions.lock().await;
2780                    let is_current = sessions
2781                        .get(&thread_id)
2782                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2783                    if is_current {
2784                        sessions.remove(&thread_id);
2785                    }
2786                }
2787            }
2788        }));
2789    }
2790
2791    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2792        self.runner_sessions.lock().await.remove(thread_id);
2793    }
2794
2795    async fn cache_runner_session(
2796        &self,
2797        thread_id: &ThreadId,
2798        destination: RunnerDestination,
2799        session: Arc<dyn RemoteRunnerSession>,
2800    ) {
2801        let cached = CachedRunnerSession {
2802            destination,
2803            session,
2804        };
2805        self.runner_sessions.lock().await.insert(
2806            thread_id.clone(),
2807            Arc::new(RunnerSessionSlot::ready(cached)),
2808        );
2809    }
2810
2811    /// Read a thread's explicit runner binding without creating or resuming a
2812    /// session. Tool routing uses this before local previews so runner-backed
2813    /// and fail-closed hosted calls never inspect the host workspace.
2814    pub(crate) async fn runner_binding_for_thread(
2815        &self,
2816        thread_id: &ThreadId,
2817    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2818        let Some(store) = &self.thread_store else {
2819            return Ok(None);
2820        };
2821        Ok(store
2822            .load_thread_metadata(thread_id)
2823            .await?
2824            .and_then(|metadata| metadata.runner_binding))
2825    }
2826
2827    /// Resolve a previously-read binding into a live remote workspace. The
2828    /// session remains lazy: callers invoke this only after policy approval.
2829    pub(crate) async fn remote_workspace_for_binding(
2830        &self,
2831        thread_id: &ThreadId,
2832        binding: ThreadRunnerBinding,
2833    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2834        let session = self
2835            .runner_session_for_thread(thread_id)
2836            .await?
2837            .map(|(_, session)| session)
2838            .ok_or_else(|| {
2839                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2840            })?;
2841        Ok(Arc::new(RemoteWorkspace {
2842            session,
2843            root: binding.workspace,
2844            read_roots: binding.read_roots,
2845        }))
2846    }
2847
2848    pub(crate) async fn runner_tool_execution_lock(
2849        &self,
2850        session: &dyn RemoteRunnerSession,
2851    ) -> Arc<RunnerToolExecutionMutex> {
2852        let state = session.state();
2853        let key = (state.provider_id, state.session_id);
2854        let mut locks = self.runner_tool_execution_locks.lock().await;
2855        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2856            return lock;
2857        }
2858
2859        locks.retain(|_, lock| lock.strong_count() > 0);
2860        let lock = Arc::new(Mutex::new(()));
2861        locks.insert(key, Arc::downgrade(&lock));
2862        lock
2863    }
2864
2865    async fn persist_runner_state(
2866        &self,
2867        thread_id: &ThreadId,
2868        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2869    ) -> anyhow::Result<()> {
2870        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2871    }
2872
2873    async fn persist_runner_state_in_store(
2874        store: Option<&Arc<dyn ThreadStore>>,
2875        thread_id: &ThreadId,
2876        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2877    ) -> anyhow::Result<()> {
2878        let Some((destination, session)) = runner else {
2879            return Ok(());
2880        };
2881        let Some(store) = store else {
2882            return Ok(());
2883        };
2884        let Some(snapshot) = store.load_thread(thread_id).await? else {
2885            return Ok(());
2886        };
2887        let Some(mut metadata) = snapshot.metadata else {
2888            return Ok(());
2889        };
2890        metadata.runner_destination = Some(destination.clone());
2891        metadata.runner_state = Some(session.state());
2892        metadata.updated_at = OffsetDateTime::now_utc();
2893        store.update_thread_metadata(metadata).await?;
2894        Ok(())
2895    }
2896
2897    fn remote_runner_provider_by_id(
2898        &self,
2899        provider_id: &str,
2900    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2901        self.registry
2902            .remote_runner_providers
2903            .iter()
2904            .find(|provider| provider.id() == provider_id)
2905            .cloned()
2906    }
2907
2908    /// Resolve the live runner session for a thread along with its provider,
2909    /// failing clearly when the thread is not runner-bound.
2910    async fn thread_runner_session(
2911        &self,
2912        thread_id: &ThreadId,
2913    ) -> anyhow::Result<(
2914        RunnerDestination,
2915        Arc<dyn RemoteRunnerProvider>,
2916        Arc<dyn RemoteRunnerSession>,
2917    )> {
2918        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2919            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2920        };
2921        let provider = self
2922            .remote_runner_provider_by_id(&destination.provider_id)
2923            .ok_or_else(|| {
2924                anyhow::anyhow!(
2925                    "remote runner provider {:?} is not installed",
2926                    destination.provider_id
2927                )
2928            })?;
2929        Ok((destination, provider, session))
2930    }
2931
2932    /// Pause a runner-bound thread's session toward standby and persist the
2933    /// post-pause state. Errors if the provider is not pausable.
2934    pub async fn pause_thread_runner(
2935        &self,
2936        thread_id: &ThreadId,
2937    ) -> anyhow::Result<RunnerSessionState> {
2938        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2939        anyhow::ensure!(
2940            provider.capabilities().pausable,
2941            "remote runner provider {:?} does not support pausing",
2942            destination.provider_id
2943        );
2944        let state = session.pause().await?;
2945        self.persist_runner_state(thread_id, Some(&(destination, session)))
2946            .await?;
2947        Ok(state)
2948    }
2949
2950    /// Resume (wake) a runner-bound thread's paused session and persist state.
2951    pub async fn resume_thread_runner(
2952        &self,
2953        thread_id: &ThreadId,
2954    ) -> anyhow::Result<RunnerSessionState> {
2955        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
2956        let state = session.resume().await?;
2957        self.persist_runner_state(thread_id, Some(&(destination, session)))
2958            .await?;
2959        Ok(state)
2960    }
2961
2962    /// Detach a runner-bound thread's session: persist the durable, rejoinable
2963    /// state and leave the remote sandbox alive. Errors if not detachable.
2964    pub async fn detach_thread_runner(
2965        &self,
2966        thread_id: &ThreadId,
2967    ) -> anyhow::Result<RunnerSessionState> {
2968        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2969        anyhow::ensure!(
2970            provider.capabilities().detachable,
2971            "remote runner provider {:?} does not support detaching",
2972            destination.provider_id
2973        );
2974        let state = session.detach().await?;
2975        // Persist the detached state explicitly so a later turn or process
2976        // rejoins the same sandbox instead of provisioning a new one.
2977        if let Some(store) = &self.thread_store
2978            && let Some(snapshot) = store.load_thread(thread_id).await?
2979            && let Some(mut metadata) = snapshot.metadata
2980        {
2981            metadata.runner_destination = Some(destination);
2982            metadata.runner_state = Some(state.clone());
2983            metadata.updated_at = OffsetDateTime::now_utc();
2984            store.update_thread_metadata(metadata).await?;
2985        }
2986        self.evict_runner_session(thread_id).await;
2987        Ok(state)
2988    }
2989
2990    /// Rejoin a previously created sandbox from a thread's persisted runner
2991    /// state without provisioning a new one. An optional `sandbox` overrides the
2992    /// persisted sandbox name (recovery by name). Persists refreshed state.
2993    pub async fn rejoin_thread_runner(
2994        &self,
2995        thread_id: &ThreadId,
2996        sandbox: Option<String>,
2997    ) -> anyhow::Result<RunnerSessionState> {
2998        let store = self
2999            .thread_store
3000            .as_ref()
3001            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3002        let metadata = store
3003            .load_thread_metadata(thread_id)
3004            .await?
3005            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3006        let destination = metadata
3007            .runner_binding
3008            .as_ref()
3009            .map(|binding| binding.destination.clone())
3010            .or_else(|| metadata.runner_destination.clone())
3011            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3012        let mut state = metadata
3013            .runner_state
3014            .clone()
3015            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3016        if let Some(sandbox) = sandbox
3017            && let Some(object) = state.metadata.as_object_mut()
3018        {
3019            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3020        }
3021        let provider = self
3022            .remote_runner_provider_by_id(&destination.provider_id)
3023            .ok_or_else(|| {
3024                anyhow::anyhow!(
3025                    "remote runner provider {:?} is not installed",
3026                    destination.provider_id
3027                )
3028            })?;
3029        let session = provider.rejoin_session(state).await?;
3030        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3031            .await?;
3032        self.cache_runner_session(thread_id, destination, session.clone())
3033            .await;
3034        Ok(session.state())
3035    }
3036
3037    async fn record_thread_usage_metadata(
3038        &self,
3039        thread_id: &ThreadId,
3040        usage: &TokenUsage,
3041    ) -> anyhow::Result<()> {
3042        if usage.is_empty() {
3043            return Ok(());
3044        }
3045        let Some(store) = &self.thread_store else {
3046            return Ok(());
3047        };
3048        let Some(snapshot) = store.load_thread(thread_id).await? else {
3049            return Ok(());
3050        };
3051        let Some(mut metadata) = snapshot.metadata else {
3052            return Ok(());
3053        };
3054        metadata
3055            .usage
3056            .get_or_insert_with(ThreadUsageMetadata::default)
3057            .add_token_usage(usage);
3058        metadata.updated_at = OffsetDateTime::now_utc();
3059        store.update_thread_metadata(metadata).await?;
3060        Ok(())
3061    }
3062
3063    pub fn start_turn(
3064        self: &Arc<Self>,
3065        mut req: StartTurnRequest,
3066    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3067        Box::pin(async move {
3068            let turn_admission = self.turn_admission.lock().await;
3069            anyhow::ensure!(
3070                self.accepting_turns.load(Ordering::Acquire),
3071                "runtime is quiescing and cannot accept new turns"
3072            );
3073            req.workspace = validate_thread_workspace(&req.workspace)?;
3074            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3075            let cfg = self.config.read().await.clone();
3076            let provider = req
3077                .provider_override
3078                .clone()
3079                .unwrap_or_else(|| cfg.default_provider.clone());
3080            self.engine_for(&provider)?;
3081            let turn_id = uuid::Uuid::new_v4().to_string();
3082            let mut initial_mailbox_ack = None;
3083            if let Some((team_id, member)) = &team_member {
3084                let pending = self
3085                    .teams
3086                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3087                    .await?;
3088                if !pending.is_empty()
3089                    && let Some(team) = self.read_team(team_id).await
3090                {
3091                    let mailbox = format_mailbox_messages(&team, &pending);
3092                    req.message = if req.message.trim().is_empty() {
3093                        mailbox
3094                    } else {
3095                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3096                    };
3097                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3098                        team_id: team_id.clone(),
3099                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3100                    });
3101                }
3102            }
3103            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3104            let drain = Arc::new(TurnDrainHandle {
3105                thread_id: req.thread_id.clone(),
3106                interrupt_requested: AtomicBool::new(false),
3107                interrupt_reason: Mutex::new(None),
3108                completed: AtomicBool::new(false),
3109                completed_notify: Notify::new(),
3110            });
3111            let active = ActiveTurnHandle {
3112                thread_id: req.thread_id.clone(),
3113                abort: abort_handle,
3114                steers: Arc::new(Mutex::new(Vec::new())),
3115                drain,
3116            };
3117            self.active_turns
3118                .write()
3119                .await
3120                .insert(turn_id.clone(), active);
3121            self.record_turn_lifecycle(
3122                req.thread_id.clone(),
3123                turn_id.clone(),
3124                TurnLifecycleState::Running,
3125                TurnCleanupState::NotRequested,
3126                None,
3127            )
3128            .await;
3129            self.active_turn_contexts.write().await.insert(
3130                turn_id.clone(),
3131                InheritedTurnContext {
3132                    workspace: req.workspace.clone(),
3133                    instructions: req.instructions.clone(),
3134                    developer_context: req.developer_context.clone(),
3135                },
3136            );
3137            if let Some((team_id, member)) = team_member {
3138                let updated = match self
3139                    .teams
3140                    .update_member(&team_id, &member.id, |member| {
3141                        member.current_turn_id = Some(turn_id.clone());
3142                        member.status = TeamMemberStatus::Running;
3143                        member.final_message = None;
3144                        member.terminal_error = None;
3145                    })
3146                    .await
3147                {
3148                    Ok(updated) => updated,
3149                    Err(error) => {
3150                        self.active_turns.write().await.remove(&turn_id);
3151                        self.active_turn_contexts.write().await.remove(&turn_id);
3152                        self.teams
3153                            .release_mailbox_reservations_for_turn(&turn_id)
3154                            .await;
3155                        return Err(error);
3156                    }
3157                };
3158                if let Some(member) = updated
3159                    .members
3160                    .into_iter()
3161                    .find(|candidate| candidate.id == member.id)
3162                {
3163                    self.emit(RoderEvent::TeamMemberStatusChanged(
3164                        TeamMemberStatusChanged {
3165                            team_id,
3166                            member_id: member.id,
3167                            member_thread_id: member.thread_id,
3168                            status: TeamMemberStatus::Running,
3169                            timestamp: OffsetDateTime::now_utc(),
3170                        },
3171                    ))
3172                    .await;
3173                }
3174            }
3175            let runtime = Arc::clone(self);
3176            let turn_req = req;
3177            let thread_id_for_task = turn_req.thread_id.clone();
3178            let turn_id_for_task = turn_id.clone();
3179            tokio::spawn(async move {
3180                let result = Abortable::new(
3181                    runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3182                    abort_registration,
3183                )
3184                .await;
3185                /*
3186                 * A failed sibling in a parallel tool batch drops in-flight external tool
3187                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3188                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3189                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3190                 * resolution; on clean completion the map holds nothing for this turn.
3191                 */
3192                runtime
3193                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3194                    .await;
3195                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3196                match &result {
3197                    Ok(Err(err)) => {
3198                        let (cleanup, ownership) =
3199                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3200                        // run_turn emits failures after the stream starts; this covers setup/startup errors.
3201                        runtime
3202                            .emit(RoderEvent::TurnFailed(TurnFailed {
3203                                thread_id: thread_id_for_task.clone(),
3204                                turn_id: turn_id_for_task.clone(),
3205                                error: err.to_string(),
3206                                error_kind: None,
3207                                usage: None,
3208                                timestamp: OffsetDateTime::now_utc(),
3209                            }))
3210                            .await;
3211                        runtime
3212                            .record_turn_lifecycle_with_ownership(
3213                                thread_id_for_task.clone(),
3214                                turn_id_for_task.clone(),
3215                                TurnLifecycleState::Failed,
3216                                cleanup,
3217                                Some(TurnLifecycleReason::ProviderFailure),
3218                                ownership,
3219                            )
3220                            .await;
3221                        let _ = runtime
3222                            .complete_team_member_turn_with_result(
3223                                &thread_id_for_task,
3224                                &turn_id_for_task,
3225                                TeamMemberStatus::Failed,
3226                                None,
3227                                Some(err.to_string()),
3228                            )
3229                            .await;
3230                    }
3231                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3232                        let (cleanup, ownership) =
3233                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3234                        runtime
3235                            .record_turn_lifecycle_with_ownership(
3236                                thread_id_for_task.clone(),
3237                                turn_id_for_task.clone(),
3238                                TurnLifecycleState::Failed,
3239                                cleanup,
3240                                Some(TurnLifecycleReason::ProviderFailure),
3241                                ownership,
3242                            )
3243                            .await;
3244                        let _ = runtime
3245                            .complete_team_member_turn_with_result(
3246                                &thread_id_for_task,
3247                                &turn_id_for_task,
3248                                TeamMemberStatus::Failed,
3249                                None,
3250                                Some("turn stopped before completion".to_string()),
3251                            )
3252                            .await;
3253                    }
3254                    Err(_) => {
3255                        let reason = if let Some(handle) = runtime
3256                            .turn_drains
3257                            .read()
3258                            .await
3259                            .get(&turn_id_for_task)
3260                            .cloned()
3261                        {
3262                            handle
3263                                .interrupt_reason
3264                                .lock()
3265                                .await
3266                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3267                        } else {
3268                            TurnLifecycleReason::RuntimeFailure
3269                        };
3270                        let (cleanup, ownership) =
3271                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3272                        runtime
3273                            .record_turn_lifecycle_with_ownership(
3274                                thread_id_for_task.clone(),
3275                                turn_id_for_task.clone(),
3276                                TurnLifecycleState::Interrupted,
3277                                cleanup,
3278                                Some(reason),
3279                                ownership,
3280                            )
3281                            .await;
3282                        runtime
3283                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3284                                thread_id: thread_id_for_task.clone(),
3285                                turn_id: turn_id_for_task.clone(),
3286                                timestamp: OffsetDateTime::now_utc(),
3287                            }))
3288                            .await;
3289                        let _ = runtime
3290                            .complete_team_member_turn_with_result(
3291                                &thread_id_for_task,
3292                                &turn_id_for_task,
3293                                TeamMemberStatus::Interrupted,
3294                                None,
3295                                None,
3296                            )
3297                            .await;
3298                    }
3299                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3300                }
3301                runtime
3302                    .teams
3303                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3304                    .await;
3305                runtime.active_turns.write().await.remove(&turn_id_for_task);
3306                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3307                    drain.completed.store(true, Ordering::Release);
3308                    drain.completed_notify.notify_waiters();
3309                }
3310                runtime
3311                    .active_turn_selections
3312                    .write()
3313                    .await
3314                    .remove(&turn_id_for_task);
3315                runtime
3316                    .active_turn_contexts
3317                    .write()
3318                    .await
3319                    .remove(&turn_id_for_task);
3320                if !completed {
3321                    // Completion paths above consume registered provider cleanup
3322                    // handles. A setup failure before an engine can stream has no
3323                    // such handle; this is a harmless final defensive sweep.
3324                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3325                        cleanups.remove(&turn_id_for_task);
3326                    });
3327                }
3328                runtime.active_turns_changed.notify_waiters();
3329                if completed {
3330                    let _ = runtime
3331                        .continue_active_goal_after_turn(thread_id_for_task)
3332                        .await;
3333                }
3334            });
3335            drop(turn_admission);
3336            Ok(turn_id)
3337        })
3338    }
3339
3340    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3341        self.active_turns
3342            .read()
3343            .await
3344            .values()
3345            .any(|handle| &handle.thread_id == thread_id)
3346    }
3347
3348    async fn ensure_codex_v2_team_capacity(
3349        &self,
3350        team: &TeamState,
3351        resuming_member_id: &str,
3352        candidate_model: Option<&str>,
3353    ) -> anyhow::Result<()> {
3354        if !is_codex_v2_team(team)
3355            && !candidate_model
3356                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3357        {
3358            return Ok(());
3359        }
3360        let active_thread_ids = self
3361            .active_turns
3362            .read()
3363            .await
3364            .values()
3365            .map(|handle| handle.thread_id.clone())
3366            .collect::<std::collections::HashSet<_>>();
3367        let resident_threads = 1 + team
3368            .members
3369            .iter()
3370            .filter(|member| {
3371                member.role != roder_api::teams::TeamMemberRole::Lead
3372                    && member.id != resuming_member_id
3373                    && active_thread_ids.contains(&member.thread_id)
3374            })
3375            .count();
3376        anyhow::ensure!(
3377            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3378            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3379            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3380        );
3381        Ok(())
3382    }
3383
3384    /// Number of currently running turns across all threads. Hosted runtime
3385    /// pools use this to avoid evicting tenants with active work.
3386    pub async fn active_turn_count(&self) -> usize {
3387        self.active_turns.read().await.len()
3388    }
3389
3390    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3391        self.active_turns
3392            .read()
3393            .await
3394            .iter()
3395            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3396    }
3397
3398    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3399        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3400        let draining_turn_id = if active_turn_id.is_none() {
3401            self.turn_drains
3402                .read()
3403                .await
3404                .iter()
3405                .find_map(|(turn_id, drain)| {
3406                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3407                })
3408        } else {
3409            None
3410        };
3411        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3412            return ThreadActivity::default();
3413        };
3414
3415        let mut active_flags = Vec::new();
3416        {
3417            let pending_approvals = self.pending_tool_approvals.lock().await;
3418            if pending_approvals
3419                .values()
3420                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3421            {
3422                active_flags.push("approvalRequired".to_string());
3423            }
3424        }
3425        {
3426            let pending_inputs = self.pending_user_inputs.lock().await;
3427            if pending_inputs
3428                .values()
3429                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3430            {
3431                active_flags.push("userInputRequired".to_string());
3432            }
3433        }
3434        {
3435            let pending_external = self.pending_external_tool_calls.lock().await;
3436            if pending_external
3437                .values()
3438                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3439            {
3440                active_flags.push("externalToolPending".to_string());
3441            }
3442        }
3443        let interrupting = self
3444            .active_turns
3445            .read()
3446            .await
3447            .get(&active_turn_id)
3448            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3449            || self
3450                .turn_drains
3451                .read()
3452                .await
3453                .get(&active_turn_id)
3454                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3455        if interrupting {
3456            active_flags.push("interrupting".to_string());
3457        }
3458        if self.pending_plan_exit().await.is_some_and(|pending| {
3459            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3460        }) {
3461            active_flags.push("planExitRequired".to_string());
3462        }
3463
3464        ThreadActivity {
3465            active_turn_id: Some(active_turn_id),
3466            active_flags,
3467        }
3468    }
3469
3470    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3471        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3472            .await
3473    }
3474
3475    async fn interrupt_turn_with_reason(
3476        &self,
3477        thread_id: ThreadId,
3478        turn_id: TurnId,
3479        reason: TurnLifecycleReason,
3480    ) -> anyhow::Result<()> {
3481        let handle = self.active_turns.write().await.remove(&turn_id);
3482        if let Some(handle) = handle {
3483            if handle
3484                .drain
3485                .interrupt_requested
3486                .swap(true, Ordering::AcqRel)
3487            {
3488                return Ok(());
3489            }
3490            *handle.drain.interrupt_reason.lock().await = Some(reason);
3491            self.turn_drains
3492                .write()
3493                .await
3494                .insert(turn_id.clone(), handle.drain.clone());
3495            let ownership = self.provider_cleanup_ownership(&turn_id);
3496            self.record_turn_lifecycle_with_ownership(
3497                thread_id.clone(),
3498                turn_id.clone(),
3499                TurnLifecycleState::InterruptRequested,
3500                TurnCleanupState::Requested,
3501                Some(reason),
3502                ownership,
3503            )
3504            .await;
3505            handle.abort.abort();
3506            self.active_turns_changed.notify_waiters();
3507        }
3508        self.active_turn_selections.write().await.remove(&turn_id);
3509        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3510            .await;
3511        Ok(())
3512    }
3513
3514    pub async fn steer_turn(
3515        &self,
3516        thread_id: ThreadId,
3517        turn_id: TurnId,
3518        message: String,
3519        images: Vec<InputImage>,
3520    ) -> anyhow::Result<()> {
3521        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3522            .await
3523    }
3524
3525    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3526        let active = self.active_turns.read().await.get(turn_id).cloned();
3527        let Some(active) = active else {
3528            return false;
3529        };
3530        let has_pending = !active.steers.lock().await.is_empty();
3531        has_pending
3532    }
3533
3534    async fn steer_turn_with_mailbox_ack(
3535        &self,
3536        thread_id: ThreadId,
3537        turn_id: TurnId,
3538        message: String,
3539        message_ids: Vec<String>,
3540        team_id: TeamId,
3541    ) -> anyhow::Result<()> {
3542        self.enqueue_turn_steer(
3543            thread_id,
3544            turn_id,
3545            message,
3546            Vec::new(),
3547            Some(MailboxDeliveryAck {
3548                team_id,
3549                message_ids,
3550            }),
3551        )
3552        .await
3553    }
3554
3555    async fn deliver_pending_team_mailbox(
3556        &self,
3557        team_id: &str,
3558        member_id: &str,
3559        member_thread_id: ThreadId,
3560        turn_id: TurnId,
3561    ) -> anyhow::Result<()> {
3562        let pending = self
3563            .teams
3564            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3565            .await?;
3566        if pending.is_empty() {
3567            return Ok(());
3568        }
3569        let message_ids = pending
3570            .iter()
3571            .map(|message| message.id.clone())
3572            .collect::<Vec<_>>();
3573        let team = self
3574            .read_team(team_id)
3575            .await
3576            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3577        if let Err(error) = self
3578            .steer_turn_with_mailbox_ack(
3579                member_thread_id,
3580                turn_id.clone(),
3581                format_mailbox_messages(&team, &pending),
3582                message_ids.clone(),
3583                team_id.to_string(),
3584            )
3585            .await
3586        {
3587            self.teams
3588                .release_mailbox_reservations(&turn_id, &message_ids)
3589                .await;
3590            return Err(error);
3591        }
3592        Ok(())
3593    }
3594
3595    async fn enqueue_turn_steer(
3596        &self,
3597        thread_id: ThreadId,
3598        turn_id: TurnId,
3599        message: String,
3600        images: Vec<InputImage>,
3601        mailbox_ack: Option<MailboxDeliveryAck>,
3602    ) -> anyhow::Result<()> {
3603        let message = message.trim().to_string();
3604        if message.is_empty() && images.is_empty() {
3605            return Ok(());
3606        }
3607
3608        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3609            anyhow::bail!("no active turn to steer");
3610        };
3611        active.steers.lock().await.push(QueuedTurnSteer {
3612            message: UserMessage::with_images(message.clone(), images),
3613            mailbox_ack,
3614        });
3615        self.emit(RoderEvent::TurnSteered(TurnSteered {
3616            thread_id,
3617            turn_id,
3618            message,
3619            timestamp: OffsetDateTime::now_utc(),
3620        }))
3621        .await;
3622        Ok(())
3623    }
3624
3625    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3626        let cfg = self.config.read().await;
3627        let model_profile =
3628            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3629        self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3630    }
3631
3632    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3633        self.registry
3634            .subagent_dispatchers
3635            .iter()
3636            .flat_map(|dispatcher| dispatcher.definitions())
3637            .collect()
3638    }
3639
3640    async fn run_turn(
3641        self: &Arc<Self>,
3642        req: StartTurnRequest,
3643        turn_id: TurnId,
3644        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3645    ) -> anyhow::Result<TurnRunOutcome> {
3646        let turn_started_at = OffsetDateTime::now_utc();
3647        self.emit(RoderEvent::TurnStarted(TurnStarted {
3648            thread_id: req.thread_id.clone(),
3649            turn_id: turn_id.clone(),
3650            runtime_profile: self.config.read().await.runtime_profile,
3651            timestamp: turn_started_at,
3652        }))
3653        .await;
3654        self.persist_turn_item(
3655            &req.thread_id,
3656            &turn_id,
3657            &TranscriptItem::UserMessage(UserMessage::with_images(
3658                req.message.clone(),
3659                req.images.clone(),
3660            )),
3661        )
3662        .await?;
3663        if let Some(ack) = initial_mailbox_ack {
3664            self.teams
3665                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3666                .await?;
3667        }
3668
3669        let mut cfg = self.config.read().await.clone();
3670        let runtime_profile = cfg.runtime_profile;
3671        let turn_deadline = turn_deadline_for_config(&cfg);
3672        let deadline_finalization_reserve =
3673            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3674        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3675        let concrete_selection = selection_mode
3676            .as_ref()
3677            .map(ModelSelectionMode::concrete_selection);
3678        let default_provider = req
3679            .provider_override
3680            .clone()
3681            .or_else(|| {
3682                concrete_selection
3683                    .as_ref()
3684                    .map(|selection| selection.provider.clone())
3685            })
3686            .unwrap_or(cfg.default_provider.clone());
3687        let default_model = req
3688            .model_override
3689            .clone()
3690            .or_else(|| {
3691                concrete_selection
3692                    .as_ref()
3693                    .map(|selection| selection.model.clone())
3694            })
3695            .unwrap_or(cfg.default_model.clone());
3696        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3697            selection_mode
3698                .as_ref()
3699                .and_then(ModelSelectionMode::reasoning)
3700        }) {
3701            validate_reasoning_effort(&default_model, reasoning)?;
3702            cfg.reasoning = Some(reasoning.to_string());
3703        }
3704        let turn_has_concrete_model_override =
3705            req.provider_override.is_some() || req.model_override.is_some();
3706        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3707            Some(ModelSelectionMode::Auto {
3708                router_id, profile, ..
3709            }) if !turn_has_concrete_model_override => (
3710                RuntimeInferenceRouterConfig {
3711                    enabled: true,
3712                    router_id: Some(router_id.clone()),
3713                },
3714                profile.clone(),
3715            ),
3716            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3717        };
3718        let mut provider = default_provider.clone();
3719        let mut model = default_model.clone();
3720        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3721        let workspace = req.workspace.clone();
3722        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3723        let mut compacted_this_turn = transcript
3724            .iter()
3725            .any(crate::compaction::is_compaction_boundary);
3726        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3727        let agent_swarm_mode_active = self
3728            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3729            .await;
3730        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3731        let mut final_assistant_text = String::new();
3732        let mut final_phase_messages = Vec::<AssistantMessage>::new();
3733        let mut final_reasoning_text = String::new();
3734        let mut final_provider_metadata = None;
3735        let mut exhausted_tool_rounds = true;
3736        let mut verification_gate =
3737            VerificationGateState::new(req.message.clone(), runtime_profile);
3738        let mut speed_policy = SpeedPolicyState::default();
3739        let mut reliability = TurnReliabilityState::default();
3740        let mut turn_usage = TokenUsage::default();
3741        // Overwritten on every inference step's Completed event so only the
3742        // terminal step's stop reason survives (mid-turn tool_use steps must
3743        // not leak onto turn/completed).
3744        let mut turn_finish_reason: Option<String> = None;
3745        let mut deadline_finalization_requested = false;
3746        let mut deadline_scoreable_completion_requested = false;
3747        let mut task_ledger_completion_reminders = 0_u8;
3748        let mut task_ledger_scoreable_checkpoints = 0_u8;
3749        let mut empty_tool_call_nudges_used = 0_u32;
3750        let mut provider_stream_retry_attempts = 0_u32;
3751        let mut routing_candidates = None;
3752        let routing_transcript_start = transcript.len().saturating_sub(1);
3753        let mut routing_escalations = 0_u32;
3754        let mut model_switch_summary_selection = None::<ModelSelection>;
3755
3756        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3757            if let Some(deadline) = turn_deadline
3758                && deadline_expired(deadline)
3759            {
3760                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3761                    .await?;
3762                return Ok(TurnRunOutcome::Stopped);
3763            }
3764            let steers = self.drain_turn_steers(&turn_id).await;
3765            self.append_steers(&req, &turn_id, &mut transcript, steers)
3766                .await?;
3767            if runtime_profile == RuntimeProfile::Eval
3768                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3769                    turn_deadline,
3770                    deadline_finalization_reserve,
3771                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3772                )
3773            {
3774                if req.task_ledger_required
3775                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3776                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3777                {
3778                    task_ledger_completion_reminders += 1;
3779                    deadline_scoreable_completion_requested = true;
3780                    let item = TranscriptItem::UserMessage(UserMessage::text(
3781                        task_ledger_deadline_completion_prompt(
3782                            remaining,
3783                            deadline_finalization_reserve,
3784                            &prompt,
3785                        ),
3786                    ));
3787                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3788                        .await?;
3789                    transcript.push(item);
3790                    continue 'tool_rounds;
3791                } else {
3792                    self.start_deadline_finalization(
3793                        &req.thread_id,
3794                        &turn_id,
3795                        &mut transcript,
3796                        remaining,
3797                    )
3798                    .await?;
3799                    deadline_finalization_requested = true;
3800                }
3801            }
3802            if runtime_profile == RuntimeProfile::Eval
3803                && req.task_ledger_required
3804                && !deadline_finalization_requested
3805                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3806                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3807                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3808                && remaining > deadline_finalization_reserve
3809                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3810            {
3811                task_ledger_scoreable_checkpoints += 1;
3812                let item = TranscriptItem::UserMessage(UserMessage::text(
3813                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3814                ));
3815                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3816                    .await?;
3817                transcript.push(item);
3818                continue 'tool_rounds;
3819            }
3820            if turn_inference_router.is_active() && routing_candidates.is_none() {
3821                routing_candidates =
3822                    Some(collect_inference_routing_candidates(&self.registry).await);
3823            }
3824            let routing_tools_model = model.clone();
3825            let routing_tools = self.filtered_tool_specs(
3826                &cfg,
3827                &model,
3828                model_profile.as_ref(),
3829                &thread_overrides.tool_allowlist,
3830                &thread_overrides.external_tools,
3831            );
3832            let prior_failures =
3833                transcript_failure_count_since(&transcript, routing_transcript_start)
3834                    .max(reliability.tool_failure_count())
3835                    .saturating_add(provider_stream_retry_attempts);
3836            let routing_selection = route_inference_selection(
3837                &self.registry,
3838                &turn_inference_router,
3839                InferenceRoutingRequest {
3840                    thread_id: &req.thread_id,
3841                    turn_id: &turn_id,
3842                    round_index: round_index as u32,
3843                    runtime_profile,
3844                    phase: speed_policy.phase(),
3845                    profile: turn_inference_router_profile.as_deref(),
3846                    default_selection: ModelSelection {
3847                        provider: default_provider.clone(),
3848                        model: default_model.clone(),
3849                    },
3850                    transcript: &transcript,
3851                    tools: &routing_tools,
3852                    candidates: routing_candidates.as_deref(),
3853                    prior_failures,
3854                    prior_escalations: routing_escalations,
3855                },
3856            )
3857            .await;
3858            if let Some(decision) = routing_selection.decision.clone() {
3859                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3860                    routing_escalations = routing_escalations.saturating_add(1);
3861                }
3862                self.emit(RoderEvent::InferenceRoutingDecision(
3863                    InferenceRoutingDecisionEvent {
3864                        thread_id: req.thread_id.clone(),
3865                        turn_id: turn_id.clone(),
3866                        round_index: round_index as u32,
3867                        default_selection: ModelSelection {
3868                            provider: default_provider.clone(),
3869                            model: default_model.clone(),
3870                        },
3871                        selected_selection: routing_selection.selection.clone(),
3872                        decision,
3873                        timestamp: OffsetDateTime::now_utc(),
3874                    },
3875                ))
3876                .await;
3877            }
3878            provider = routing_selection.selection.provider.clone();
3879            model = routing_selection.selection.model.clone();
3880            let engine = self.engine_for(&provider)?;
3881            let capabilities = engine.capabilities();
3882            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3883            let tools = if capabilities.tool_calls {
3884                if model == routing_tools_model {
3885                    routing_tools.clone()
3886                } else {
3887                    self.filtered_tool_specs(
3888                        &cfg,
3889                        &model,
3890                        model_profile.as_ref(),
3891                        &thread_overrides.tool_allowlist,
3892                        &thread_overrides.external_tools,
3893                    )
3894                }
3895            } else {
3896                Vec::new()
3897            };
3898            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3899            let tool_choice = if tools.is_empty() {
3900                ToolChoice::None
3901            } else {
3902                ToolChoice::Auto
3903            };
3904            let summary_selection = ModelSelection {
3905                provider: provider.clone(),
3906                model: model.clone(),
3907            };
3908            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3909                if let Some(summary) = model_switch_summary(
3910                    &transcript,
3911                    model_profile.as_ref(),
3912                    &provider,
3913                    &model,
3914                    &tools,
3915                ) {
3916                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3917                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3918                        .await?;
3919                    transcript.push(item);
3920                }
3921                model_switch_summary_selection = Some(summary_selection);
3922            }
3923
3924            if !capabilities.image_input && transcript_has_images(&transcript) {
3925                self.fail_turn_with_error(
3926                    &req.thread_id,
3927                    &turn_id,
3928                    format!("provider {provider} does not support image input"),
3929                )
3930                .await?;
3931                return Ok(TurnRunOutcome::Stopped);
3932            }
3933            transcript = self
3934                .compact_transcript_if_needed(
3935                    &req.thread_id,
3936                    &turn_id,
3937                    &provider,
3938                    &model,
3939                    transcript,
3940                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3941                )
3942                .await?;
3943            compacted_this_turn = compacted_this_turn
3944                || transcript
3945                    .iter()
3946                    .any(crate::compaction::is_compaction_boundary);
3947
3948            let speed_policy_decision =
3949                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3950            let request_reasoning = reasoning_from_decision(
3951                speed_policy_decision.as_ref(),
3952                routing_selection
3953                    .reasoning
3954                    .clone()
3955                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3956            );
3957            self.active_turn_selections.write().await.insert(
3958                turn_id.clone(),
3959                ModelSelectionMode::manual(
3960                    provider.clone(),
3961                    model.clone(),
3962                    request_reasoning.level.clone(),
3963                ),
3964            );
3965            if let Some(limit) = reliability.record_model_call(
3966                &cfg.reliability,
3967                runtime_profile == RuntimeProfile::Interactive,
3968            ) {
3969                self.fail_turn_due_to_reliability_limit(
3970                    &req.thread_id,
3971                    &turn_id,
3972                    &provider,
3973                    &model,
3974                    limit,
3975                    &transcript,
3976                )
3977                .await?;
3978                return Ok(TurnRunOutcome::Stopped);
3979            }
3980            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3981                thread_id: req.thread_id.clone(),
3982                turn_id: turn_id.clone(),
3983                engine_id: engine.id(),
3984                model: ModelSelection {
3985                    provider: provider.clone(),
3986                    model: model.clone(),
3987                },
3988                reasoning: request_reasoning.clone(),
3989                speed_policy: speed_policy_decision.clone(),
3990                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3991                timestamp: OffsetDateTime::now_utc(),
3992            }))
3993            .await;
3994
3995            let mut instructions = req.instructions.clone();
3996            if let Some(extra) = &thread_overrides.developer_instructions {
3997                instructions = apply_thread_developer_instructions(instructions, extra);
3998            }
3999            if let Some(context) = req.developer_context.as_deref() {
4000                instructions = apply_turn_developer_context(instructions, context);
4001            }
4002            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4003            if let Some(profile) = &model_profile {
4004                instructions = apply_model_instruction_overlay(instructions, profile);
4005            }
4006            if req.task_ledger_required
4007                && runtime_profile == RuntimeProfile::Eval
4008                && !transcript_has_task_ledger(&transcript)
4009            {
4010                instructions = apply_task_ledger_required(instructions);
4011            }
4012            if effective_policy_mode == PolicyMode::Plan {
4013                instructions = apply_plan_mode(instructions);
4014            }
4015            if agent_swarm_mode_active {
4016                instructions = apply_agent_swarm_mode(instructions);
4017            }
4018            if model_supports_reasoning_effort(&model, REASONING_ULTRA) {
4019                instructions = apply_codex_multi_agent_mode(
4020                    instructions,
4021                    request_reasoning.level.as_deref() == Some(REASONING_ULTRA),
4022                );
4023            }
4024            instructions = self
4025                .goals
4026                .apply_goal_instructions(&req.thread_id, instructions)
4027                .await?;
4028            let mut request_metadata = serde_json::json!({});
4029            if let Some(decision) = &speed_policy_decision {
4030                request_metadata["speedPolicy"] = serde_json::json!(decision);
4031            }
4032            if let Some(decision) = routing_selection.decision.as_ref() {
4033                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4034            }
4035            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4036                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4037            }
4038            if let Some(profile) = &model_profile {
4039                request_metadata["modelProfile"] = serde_json::json!({
4040                    "model": profile.model,
4041                    "providerFamily": profile.provider_family,
4042                    "editTool": profile.edit_tool,
4043                    "schemaPolicy": profile.schema_policy,
4044                    "instructionOverlay": profile.instruction_overlay,
4045                    "parallelToolCalls": profile.parallel_tool_calls,
4046                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4047                });
4048            }
4049            let task_ledger_required_this_round = req.task_ledger_required
4050                && runtime_profile == RuntimeProfile::Eval
4051                && !deadline_finalization_requested
4052                && !transcript_has_task_ledger(&transcript);
4053            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4054                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4055                .filter(|tools| !tools.is_empty());
4056            let request_tools = if deadline_finalization_requested {
4057                Vec::new()
4058            } else if let Some(ledger_tools) = &task_ledger_tools {
4059                ledger_tools.clone()
4060            } else {
4061                tools.clone()
4062            };
4063            let request_tool_choice = if deadline_finalization_requested {
4064                ToolChoice::None
4065            } else if task_ledger_tools.is_some() {
4066                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4067            } else {
4068                tool_choice.clone()
4069            };
4070            if deadline_finalization_requested {
4071                request_metadata["deadlineFinalization"] = serde_json::json!({
4072                    "reserveSeconds": deadline_finalization_reserve,
4073                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4074                });
4075            } else if deadline_scoreable_completion_requested {
4076                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4077                    "reserveSeconds": deadline_finalization_reserve,
4078                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4079                });
4080            }
4081            let request = AgentInferenceRequest {
4082                model: ModelSelection {
4083                    provider: provider.clone(),
4084                    model: model.clone(),
4085                },
4086                instructions,
4087                transcript: transcript.clone(),
4088                tools: request_tools,
4089                tool_choice: request_tool_choice,
4090                reasoning: request_reasoning,
4091                output: OutputConfig::default(),
4092                runtime: RuntimeHints {
4093                    auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4094                    profile: runtime_profile,
4095                    parallel_tool_calls: Some(parallel_tool_calls),
4096                    hosted_web_search: cfg.hosted_web_search.clone(),
4097                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4098                    speed_policy: speed_policy_decision,
4099                    reliability: Some(cfg.reliability.clone().into()),
4100                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4101                    ..RuntimeHints::default()
4102                },
4103                metadata: request_metadata,
4104            };
4105
4106            let ctx = InferenceTurnContext {
4107                thread_id: &req.thread_id,
4108                turn_id: &turn_id,
4109                tool_executor: Some(std::sync::Arc::new(
4110                    crate::tool_execution::RuntimeTurnToolExecutor {
4111                        runtime: Arc::clone(self),
4112                        thread_id: req.thread_id.clone(),
4113                        turn_id: turn_id.clone(),
4114                        workspace: Some(workspace.clone()),
4115                        deadline: turn_deadline,
4116                    },
4117                )),
4118            };
4119            let stream_future = engine.stream_turn(ctx, request);
4120            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4121                turn_deadline,
4122                runtime_profile,
4123                req.task_ledger_required,
4124                deadline_finalization_reserve,
4125                deadline_finalization_requested || deadline_scoreable_completion_requested,
4126                task_ledger_scoreable_checkpoints,
4127                &transcript,
4128            ) {
4129                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4130                    Ok(stream) => stream?,
4131                    Err(_) => {
4132                        if runtime_profile == RuntimeProfile::Eval
4133                            && !deadline_finalization_requested
4134                        {
4135                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4136                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4137                                && task_ledger_scoreable_checkpoints
4138                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4139                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4140                            {
4141                                task_ledger_scoreable_checkpoints += 1;
4142                                let item = TranscriptItem::UserMessage(UserMessage::text(
4143                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4144                                ));
4145                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4146                                    .await?;
4147                                transcript.push(item);
4148                                continue 'tool_rounds;
4149                            }
4150                            self.start_deadline_finalization(
4151                                &req.thread_id,
4152                                &turn_id,
4153                                &mut transcript,
4154                                remaining,
4155                            )
4156                            .await?;
4157                            deadline_finalization_requested = true;
4158                            continue 'tool_rounds;
4159                        }
4160                        self.fail_turn_due_to_deadline(
4161                            &req.thread_id,
4162                            &turn_id,
4163                            deadline,
4164                            &transcript,
4165                        )
4166                        .await?;
4167                        return Ok(TurnRunOutcome::Stopped);
4168                    }
4169                }
4170            } else {
4171                stream_future.await?
4172            };
4173            let mut assistant_text = String::new();
4174            let mut phase_messages = Vec::<AssistantMessage>::new();
4175            let mut reasoning_text = String::new();
4176            let mut tool_calls = Vec::new();
4177            let mut provider_metadata = None;
4178            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4179            // sometimes abort the SSE stream after emitting the compaction item
4180            // ("error decoding response body") before response.completed, so we
4181            // must not rely solely on ProviderMetadata for the boundary.
4182            let mut stream_compaction_item: Option<serde_json::Value> = None;
4183
4184            loop {
4185                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4186                    turn_deadline,
4187                    runtime_profile,
4188                    req.task_ledger_required,
4189                    deadline_finalization_reserve,
4190                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4191                    task_ledger_scoreable_checkpoints,
4192                    &transcript,
4193                ) {
4194                    match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4195                        Ok(next) => next,
4196                        Err(_) => {
4197                            if runtime_profile == RuntimeProfile::Eval
4198                                && !deadline_finalization_requested
4199                            {
4200                                let remaining =
4201                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4202                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4203                                    && task_ledger_scoreable_checkpoints
4204                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4205                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4206                                {
4207                                    task_ledger_scoreable_checkpoints += 1;
4208                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4209                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4210                                    ));
4211                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4212                                        .await?;
4213                                    transcript.push(item);
4214                                    continue 'tool_rounds;
4215                                }
4216                                self.start_deadline_finalization(
4217                                    &req.thread_id,
4218                                    &turn_id,
4219                                    &mut transcript,
4220                                    remaining,
4221                                )
4222                                .await?;
4223                                deadline_finalization_requested = true;
4224                                continue 'tool_rounds;
4225                            }
4226                            self.fail_turn_due_to_deadline(
4227                                &req.thread_id,
4228                                &turn_id,
4229                                deadline,
4230                                &transcript,
4231                            )
4232                            .await?;
4233                            return Ok(TurnRunOutcome::Stopped);
4234                        }
4235                    }
4236                } else {
4237                    stream.next().await
4238                };
4239                let Some(res) = next else {
4240                    break;
4241                };
4242                let event = match res {
4243                    Ok(event) => event,
4244                    Err(err) => {
4245                        let error = err.to_string();
4246                        if runtime_profile == RuntimeProfile::Eval
4247                            && !deadline_finalization_requested
4248                            && let Some(cause) = provider_stream_retry_cause(&error)
4249                        {
4250                            let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4251                            let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4252                            if retry_attempt < policy.provider_retry_max_attempts {
4253                                provider_stream_retry_attempts = retry_attempt;
4254                                let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4255                                self.emit(RoderEvent::ReliabilityRetryRecorded(
4256                                    ReliabilityRetryRecorded {
4257                                        context: ReliabilityContext {
4258                                            thread_id: req.thread_id.clone(),
4259                                            turn_id: turn_id.clone(),
4260                                            provider: Some(provider.clone()),
4261                                            model: Some(model.clone()),
4262                                            ..ReliabilityContext::default()
4263                                        },
4264                                        error_class: ReliabilityErrorClass::ProviderError,
4265                                        decision: ReliabilityRetryDecision::Retry,
4266                                        attempt: retry_attempt,
4267                                        max_attempts: policy.provider_retry_max_attempts,
4268                                        delay_ms: Some(delay_ms),
4269                                        details: ReliabilityDetails::redacted(format!(
4270                                            "{cause}: {error}"
4271                                        )),
4272                                        timestamp: OffsetDateTime::now_utc(),
4273                                    },
4274                                ))
4275                                .await;
4276                                if delay_ms > 0 {
4277                                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4278                                        .await;
4279                                }
4280                                continue 'tool_rounds;
4281                            }
4282                        }
4283                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4284                            thread_id: req.thread_id.clone(),
4285                            turn_id: turn_id.clone(),
4286                            error: error.clone(),
4287                            error_kind: None,
4288                            usage: None,
4289                            timestamp: OffsetDateTime::now_utc(),
4290                        }))
4291                        .await;
4292                        self.complete_team_member_turn_with_result(
4293                            &req.thread_id,
4294                            &turn_id,
4295                            TeamMemberStatus::Failed,
4296                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4297                            Some(error),
4298                        )
4299                        .await?;
4300                        return Err(err);
4301                    }
4302                };
4303
4304                let inference_timestamp = OffsetDateTime::now_utc();
4305                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4306                    thread_id: req.thread_id.clone(),
4307                    turn_id: turn_id.clone(),
4308                    event: event.clone(),
4309                    timestamp: inference_timestamp,
4310                }))
4311                .await;
4312
4313                match event {
4314                    InferenceEvent::MessageDelta(delta) => {
4315                        if let Some((team_id, member)) =
4316                            self.teams.member_for_thread(&req.thread_id).await
4317                        {
4318                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4319                                team_id,
4320                                member_id: member.id,
4321                                member_thread_id: req.thread_id.clone(),
4322                                turn_id: turn_id.clone(),
4323                                delta: delta.text.clone(),
4324                                timestamp: OffsetDateTime::now_utc(),
4325                            }))
4326                            .await;
4327                        }
4328                        if is_final_answer_phase(delta.phase.as_deref()) {
4329                            assistant_text.push_str(&delta.text);
4330                        } else if let Some(last) = phase_messages.last_mut()
4331                            && last.phase == delta.phase
4332                        {
4333                            last.text.push_str(&delta.text);
4334                        } else {
4335                            phase_messages.push(AssistantMessage {
4336                                text: delta.text,
4337                                phase: delta.phase,
4338                            });
4339                        }
4340                    }
4341                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4342                    InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4343                    InferenceEvent::Failed(failure) => {
4344                        speed_policy.record_failure();
4345                        let error = failure.message;
4346                        self.persist_turn_item(
4347                            &req.thread_id,
4348                            &turn_id,
4349                            &TranscriptItem::Error(ErrorRecord {
4350                                message: error.clone(),
4351                            }),
4352                        )
4353                        .await?;
4354                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4355                            thread_id: req.thread_id.clone(),
4356                            turn_id: turn_id.clone(),
4357                            error: error.clone(),
4358                            error_kind: None,
4359                            usage: None,
4360                            timestamp: OffsetDateTime::now_utc(),
4361                        }))
4362                        .await;
4363                        self.complete_team_member_turn_with_result(
4364                            &req.thread_id,
4365                            &turn_id,
4366                            TeamMemberStatus::Failed,
4367                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4368                            Some(error),
4369                        )
4370                        .await?;
4371                        return Ok(TurnRunOutcome::Stopped);
4372                    }
4373                    InferenceEvent::Usage(usage) => {
4374                        turn_usage.add_assign(&usage);
4375                    }
4376                    InferenceEvent::Completed(metadata) => {
4377                        turn_finish_reason = metadata
4378                            .stop_reason
4379                            .as_deref()
4380                            .map(finish_reason_from_stop_reason);
4381                    }
4382                    InferenceEvent::Compaction(progress) => {
4383                        // Persist the boundary as soon as the provider emits a
4384                        // completed compaction item. ChatGPT Codex often aborts
4385                        // the SSE stream right after ("error decoding response
4386                        // body"), so waiting for ProviderMetadata loses the
4387                        // boundary and every subsequent request re-compacts.
4388                        if progress.status == "completed"
4389                            && let Some(item) = progress.item
4390                        {
4391                            let already = stream_compaction_item
4392                                .as_ref()
4393                                .and_then(|existing| {
4394                                    existing.get("id").and_then(serde_json::Value::as_str)
4395                                })
4396                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4397                                .is_some_and(|(a, b)| a == b);
4398                            if !already {
4399                                stream_compaction_item = Some(item.clone());
4400                                let boundary = TranscriptItem::ProviderMetadata(
4401                                    crate::compaction::provider_metadata_from_compaction_item(item),
4402                                );
4403                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4404                                    .await?;
4405                                transcript.push(boundary);
4406                                transcript =
4407                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4408                                compacted_this_turn = true;
4409                            }
4410                        }
4411                    }
4412                    InferenceEvent::HostedToolCallStarted(_)
4413                    | InferenceEvent::HostedToolCallCompleted(_)
4414                    | InferenceEvent::ToolCallStarted(_)
4415                    | InferenceEvent::ToolCallDelta(_) => {}
4416                    InferenceEvent::ProviderMetadata(mut metadata) => {
4417                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4418                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4419                                &mut metadata,
4420                                compaction_item,
4421                            );
4422                        }
4423                        provider_metadata = Some(metadata);
4424                    }
4425                }
4426            }
4427
4428            speed_policy.record_model_output(
4429                !assistant_text.is_empty() || !phase_messages.is_empty(),
4430                tool_calls.len(),
4431            );
4432            if tool_calls.is_empty() {
4433                let steers = self.drain_turn_steers(&turn_id).await;
4434                if !steers.is_empty() {
4435                    for message in phase_messages {
4436                        let item = TranscriptItem::AssistantMessage(message);
4437                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4438                            .await?;
4439                        transcript.push(item);
4440                        self.persist_model_profile_segment(
4441                            &req.thread_id,
4442                            &turn_id,
4443                            model_profile.as_ref(),
4444                            &provider,
4445                            &model,
4446                            "assistant",
4447                        )
4448                        .await?;
4449                    }
4450                    if !assistant_text.is_empty() {
4451                        let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4452                            text: assistant_text,
4453                            phase: Some(FINAL_ANSWER_PHASE.to_string()),
4454                        });
4455                        self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4456                            .await?;
4457                        transcript.push(assistant);
4458                        self.persist_model_profile_segment(
4459                            &req.thread_id,
4460                            &turn_id,
4461                            model_profile.as_ref(),
4462                            &provider,
4463                            &model,
4464                            "assistant",
4465                        )
4466                        .await?;
4467                    }
4468                    if let Some(metadata) = provider_metadata {
4469                        let had_provider_compaction =
4470                            crate::compaction::provider_metadata_has_compaction(&metadata);
4471                        let item = TranscriptItem::ProviderMetadata(metadata);
4472                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4473                            .await?;
4474                        transcript.push(item);
4475                        if had_provider_compaction {
4476                            // Server-side compaction replaced the prior window;
4477                            // drop pre-boundary items so the next request does
4478                            // not re-send (and re-compact) the full history.
4479                            transcript =
4480                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4481                            compacted_this_turn = true;
4482                        }
4483                    }
4484                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4485                        .await?;
4486                    continue;
4487                }
4488                if !deadline_finalization_requested
4489                    && req.task_ledger_required
4490                    && runtime_profile == RuntimeProfile::Eval
4491                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4492                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4493                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4494                {
4495                    task_ledger_completion_reminders += 1;
4496                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4497                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4498                        .await?;
4499                    transcript.push(item);
4500                    continue;
4501                }
4502                if !deadline_finalization_requested
4503                    && let Some(prompt) = verification_gate.blocking_prompt()
4504                {
4505                    speed_policy.record_verification_required();
4506                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4507                        thread_id: req.thread_id.clone(),
4508                        turn_id: turn_id.clone(),
4509                        reason: verification_gate.reason(),
4510                        changed_files: verification_gate.changed_files(),
4511                        tool_evidence: verification_gate.tool_evidence.clone(),
4512                        tests_run: verification_gate.tests_run.clone(),
4513                        open_gaps: verification_gate.open_gaps.clone(),
4514                        timestamp: OffsetDateTime::now_utc(),
4515                    }))
4516                    .await;
4517                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4518                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4519                        .await?;
4520                    transcript.push(item);
4521                    continue;
4522                }
4523                // Loop persistence nudge: in non-interactive/eval runs, when the
4524                // model returns a final message with no tool calls, gently prompt
4525                // it to keep going (bounded by `empty_tool_call_nudges`) before
4526                // ending the turn. Gated to non-interactive/eval so interactive
4527                // chat completions are never delayed, and only fires when the
4528                // model actually produced a final answer to re-examine.
4529                if !deadline_finalization_requested
4530                    && runtime_profile != RuntimeProfile::Interactive
4531                    && cfg.reliability.empty_tool_call_nudges > 0
4532                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4533                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4534                {
4535                    empty_tool_call_nudges_used += 1;
4536                    let item = TranscriptItem::UserMessage(UserMessage::text(
4537                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4538                    ));
4539                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4540                        .await?;
4541                    transcript.push(item);
4542                    continue;
4543                }
4544                if deadline_finalization_requested
4545                    && assistant_text.trim().is_empty()
4546                    && phase_messages.is_empty()
4547                {
4548                    assistant_text = format!(
4549                        "Deadline finalization completed without model text. {}",
4550                        turn_partial_result(&transcript)
4551                    );
4552                }
4553                final_phase_messages = phase_messages;
4554                final_assistant_text = assistant_text;
4555                final_reasoning_text = reasoning_text;
4556                final_provider_metadata = provider_metadata;
4557                exhausted_tool_rounds = false;
4558                break;
4559            }
4560
4561            for message in phase_messages {
4562                let item = TranscriptItem::AssistantMessage(message);
4563                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4564                    .await?;
4565                transcript.push(item);
4566                self.persist_model_profile_segment(
4567                    &req.thread_id,
4568                    &turn_id,
4569                    model_profile.as_ref(),
4570                    &provider,
4571                    &model,
4572                    "assistant",
4573                )
4574                .await?;
4575            }
4576            if !assistant_text.is_empty() {
4577                transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4578                    text: assistant_text,
4579                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4580                }));
4581                self.persist_model_profile_segment(
4582                    &req.thread_id,
4583                    &turn_id,
4584                    model_profile.as_ref(),
4585                    &provider,
4586                    &model,
4587                    "assistant",
4588                )
4589                .await?;
4590            }
4591            if let Some(metadata) = provider_metadata {
4592                let had_provider_compaction =
4593                    crate::compaction::provider_metadata_has_compaction(&metadata);
4594                let item = TranscriptItem::ProviderMetadata(metadata);
4595                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4596                    .await?;
4597                transcript.push(item);
4598                if had_provider_compaction {
4599                    // Server-side compaction replaced the prior window; drop
4600                    // pre-boundary items so subsequent tool rounds use the
4601                    // compact window as the next input.
4602                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4603                    compacted_this_turn = true;
4604                }
4605            }
4606            for call in &tool_calls {
4607                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4608                    id: call.id.clone(),
4609                    name: call.name.clone(),
4610                    arguments: call.arguments.clone(),
4611                });
4612                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4613                    .await?;
4614                transcript.push(tool_item);
4615                self.persist_model_profile_segment(
4616                    &req.thread_id,
4617                    &turn_id,
4618                    model_profile.as_ref(),
4619                    &provider,
4620                    &model,
4621                    "tool_call",
4622                )
4623                .await?;
4624            }
4625            if let Some(deadline) = turn_deadline
4626                && deadline_expired(deadline)
4627            {
4628                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4629                    .await?;
4630                return Ok(TurnRunOutcome::Stopped);
4631            }
4632            let results = self
4633                .route_tool_calls(
4634                    &req.thread_id,
4635                    &turn_id,
4636                    tool_calls,
4637                    parallel_tool_calls,
4638                    Some(workspace.as_str()),
4639                    turn_deadline,
4640                )
4641                .await?;
4642            let reliability_limit = reliability.record_tool_results(
4643                &cfg.reliability,
4644                &results,
4645                runtime_profile == RuntimeProfile::Interactive,
4646            );
4647            for result in results {
4648                verification_gate.record_tool_result(&result);
4649                transcript.push(TranscriptItem::ToolResult(result));
4650                self.persist_model_profile_segment(
4651                    &req.thread_id,
4652                    &turn_id,
4653                    model_profile.as_ref(),
4654                    &provider,
4655                    &model,
4656                    "tool_result",
4657                )
4658                .await?;
4659            }
4660            if let Some(limit) = reliability_limit {
4661                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4662                    // Loop persistence: rather than ending the turn, reset the
4663                    // consecutive-failure counter, nudge the model to keep going,
4664                    // and continue the round loop. Bounded by the per-turn tool
4665                    // failure ceiling, `max_model_calls_per_turn`, and
4666                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4667                    self.record_reliability_limit_continuation(
4668                        &req.thread_id,
4669                        &turn_id,
4670                        &provider,
4671                        &model,
4672                        limit,
4673                    )
4674                    .await;
4675                    reliability.reset_consecutive_failures();
4676                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4677                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4678                    ));
4679                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4680                        .await?;
4681                    transcript.push(nudge);
4682                } else {
4683                    self.fail_turn_due_to_reliability_limit(
4684                        &req.thread_id,
4685                        &turn_id,
4686                        &provider,
4687                        &model,
4688                        limit,
4689                        &transcript,
4690                    )
4691                    .await?;
4692                    return Ok(TurnRunOutcome::Stopped);
4693                }
4694            }
4695            transcript = self
4696                .compact_transcript_if_needed(
4697                    &req.thread_id,
4698                    &turn_id,
4699                    &provider,
4700                    &model,
4701                    transcript,
4702                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4703                )
4704                .await?;
4705            compacted_this_turn = compacted_this_turn
4706                || transcript
4707                    .iter()
4708                    .any(crate::compaction::is_compaction_boundary);
4709        }
4710
4711        if exhausted_tool_rounds {
4712            let message =
4713                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4714            self.persist_turn_item(
4715                &req.thread_id,
4716                &turn_id,
4717                &TranscriptItem::Error(ErrorRecord {
4718                    message: message.clone(),
4719                }),
4720            )
4721            .await?;
4722            self.emit(RoderEvent::TurnFailed(TurnFailed {
4723                thread_id: req.thread_id.clone(),
4724                turn_id: turn_id.clone(),
4725                error: message.clone(),
4726                error_kind: None,
4727                usage: None,
4728                timestamp: OffsetDateTime::now_utc(),
4729            }))
4730            .await;
4731            self.complete_team_member_turn_with_result(
4732                &req.thread_id,
4733                &turn_id,
4734                TeamMemberStatus::Failed,
4735                None,
4736                Some(message),
4737            )
4738            .await?;
4739            return Ok(TurnRunOutcome::Stopped);
4740        }
4741
4742        if !final_reasoning_text.is_empty() {
4743            self.persist_turn_item(
4744                &req.thread_id,
4745                &turn_id,
4746                &TranscriptItem::ReasoningSummary(ReasoningSummary {
4747                    text: final_reasoning_text,
4748                }),
4749            )
4750            .await?;
4751        }
4752        for message in final_phase_messages {
4753            self.persist_turn_item(
4754                &req.thread_id,
4755                &turn_id,
4756                &TranscriptItem::AssistantMessage(message),
4757            )
4758            .await?;
4759            self.persist_model_profile_segment(
4760                &req.thread_id,
4761                &turn_id,
4762                model_profile.as_ref(),
4763                &provider,
4764                &model,
4765                "assistant",
4766            )
4767            .await?;
4768        }
4769        if !final_assistant_text.is_empty() {
4770            self.persist_turn_item(
4771                &req.thread_id,
4772                &turn_id,
4773                &TranscriptItem::AssistantMessage(AssistantMessage {
4774                    text: final_assistant_text.clone(),
4775                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4776                }),
4777            )
4778            .await?;
4779            self.persist_model_profile_segment(
4780                &req.thread_id,
4781                &turn_id,
4782                model_profile.as_ref(),
4783                &provider,
4784                &model,
4785                "assistant",
4786            )
4787            .await?;
4788        }
4789        if let Some(metadata) = final_provider_metadata {
4790            self.persist_turn_item(
4791                &req.thread_id,
4792                &turn_id,
4793                &TranscriptItem::ProviderMetadata(metadata),
4794            )
4795            .await?;
4796        }
4797
4798        let turn_usage_tokens = turn_usage.total_tokens as i64;
4799        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4800        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4801            .await?;
4802        self.goals
4803            .account_turn_usage(
4804                &req.thread_id,
4805                turn_usage_tokens,
4806                OffsetDateTime::now_utc() - turn_started_at,
4807            )
4808            .await?;
4809        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4810        self.record_turn_lifecycle_with_ownership(
4811            req.thread_id.clone(),
4812            turn_id.clone(),
4813            TurnLifecycleState::Completed,
4814            cleanup,
4815            None,
4816            ownership,
4817        )
4818        .await;
4819        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4820            thread_id: req.thread_id.clone(),
4821            turn_id: turn_id.clone(),
4822            usage: completed_usage,
4823            finish_reason: turn_finish_reason,
4824            timestamp: OffsetDateTime::now_utc(),
4825        }))
4826        .await;
4827        self.complete_team_member_turn_with_result(
4828            &req.thread_id,
4829            &turn_id,
4830            TeamMemberStatus::Completed,
4831            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4832            None,
4833        )
4834        .await?;
4835        Ok(TurnRunOutcome::Completed)
4836    }
4837
4838    async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
4839        let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
4840            return Vec::new();
4841        };
4842        let mut steers = active.steers.lock().await;
4843        std::mem::take(&mut *steers)
4844    }
4845
4846    async fn route_tool_calls(
4847        self: &Arc<Self>,
4848        thread_id: &ThreadId,
4849        turn_id: &TurnId,
4850        calls: Vec<ToolCallCompleted>,
4851        parallel: bool,
4852        workspace: Option<&str>,
4853        deadline: Option<OffsetDateTime>,
4854    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4855        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4856        // `agent_swarm` must be the only tool call in a model response. A mixed
4857        // or multi-swarm batch is denied wholesale with actionable retry text so
4858        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4859        // still gets a response (keeping the chat-completions transcript valid).
4860        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4861            calls.iter().map(|call| call.name.as_str()),
4862        ) {
4863            let message = violation.deny_message();
4864            return Ok(calls
4865                .into_iter()
4866                .map(|call| ToolResultRecord {
4867                    id: call.id,
4868                    name: Some(call.name),
4869                    result: message.clone(),
4870                    display_payload: None,
4871                    is_error: true,
4872                })
4873                .collect());
4874        }
4875        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4876        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4877        // progress flows through the existing Subagent* trace events.
4878        let swarm_call = (calls.len() == 1
4879            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4880            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4881        if let Some((tool_id, args)) = &swarm_call {
4882            self.emit(RoderEvent::AgentSwarmStarted(
4883                roder_api::subagents::AgentSwarmStarted {
4884                    thread_id: thread_id.clone(),
4885                    turn_id: turn_id.clone(),
4886                    tool_id: tool_id.clone(),
4887                    child_count: agent_swarm_child_count(args),
4888                    timestamp: OffsetDateTime::now_utc(),
4889                },
4890            ))
4891            .await;
4892        }
4893
4894        let force_sequential = calls
4895            .iter()
4896            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4897        let results =
4898            if parallel && !force_sequential {
4899                try_join_all(calls.into_iter().map(|call| {
4900                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4901                }))
4902                .await
4903            } else {
4904                let mut results = Vec::with_capacity(calls.len());
4905                for call in calls {
4906                    results.push(
4907                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4908                            .await?,
4909                    );
4910                }
4911                Ok(results)
4912            }?;
4913
4914        if let Some((tool_id, _)) = &swarm_call
4915            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4916            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4917        {
4918            self.emit(RoderEvent::AgentSwarmCompleted(
4919                roder_api::subagents::AgentSwarmCompleted {
4920                    thread_id: thread_id.clone(),
4921                    turn_id: turn_id.clone(),
4922                    tool_id: tool_id.clone(),
4923                    completed,
4924                    failed,
4925                    aborted,
4926                    timestamp: OffsetDateTime::now_utc(),
4927                },
4928            ))
4929            .await;
4930        }
4931
4932        Ok(results)
4933    }
4934
4935    async fn fail_turn_with_error(
4936        &self,
4937        thread_id: &ThreadId,
4938        turn_id: &TurnId,
4939        message: String,
4940    ) -> anyhow::Result<()> {
4941        self.persist_turn_item(
4942            thread_id,
4943            turn_id,
4944            &TranscriptItem::Error(ErrorRecord {
4945                message: message.clone(),
4946            }),
4947        )
4948        .await?;
4949        self.emit(RoderEvent::TurnFailed(TurnFailed {
4950            thread_id: thread_id.clone(),
4951            turn_id: turn_id.clone(),
4952            error: message.clone(),
4953            error_kind: None,
4954            usage: None,
4955            timestamp: OffsetDateTime::now_utc(),
4956        }))
4957        .await;
4958        self.complete_team_member_turn_with_result(
4959            thread_id,
4960            turn_id,
4961            TeamMemberStatus::Failed,
4962            None,
4963            Some(message),
4964        )
4965        .await?;
4966        Ok(())
4967    }
4968
4969    async fn fail_turn_due_to_deadline(
4970        &self,
4971        thread_id: &ThreadId,
4972        turn_id: &TurnId,
4973        deadline: OffsetDateTime,
4974        transcript: &[TranscriptItem],
4975    ) -> anyhow::Result<()> {
4976        let partial_result = turn_partial_result(transcript);
4977        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
4978            thread_id: thread_id.clone(),
4979            turn_id: turn_id.clone(),
4980            summary: partial_result.clone(),
4981            timestamp: OffsetDateTime::now_utc(),
4982        }))
4983        .await;
4984        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
4985            thread_id: thread_id.clone(),
4986            turn_id: turn_id.clone(),
4987            deadline,
4988            partial_result: partial_result.clone(),
4989            timestamp: OffsetDateTime::now_utc(),
4990        }))
4991        .await;
4992        let message = "turn deadline expired".to_string();
4993        self.persist_turn_item(
4994            thread_id,
4995            turn_id,
4996            &TranscriptItem::Error(ErrorRecord {
4997                message: format!("{message}: {partial_result}"),
4998            }),
4999        )
5000        .await?;
5001        self.emit(RoderEvent::TurnFailed(TurnFailed {
5002            thread_id: thread_id.clone(),
5003            turn_id: turn_id.clone(),
5004            error: message.clone(),
5005            error_kind: Some("deadline_timeout".to_string()),
5006            usage: None,
5007            timestamp: OffsetDateTime::now_utc(),
5008        }))
5009        .await;
5010        self.complete_team_member_turn_with_result(
5011            thread_id,
5012            turn_id,
5013            TeamMemberStatus::Failed,
5014            None,
5015            Some(format!("{message}: {partial_result}")),
5016        )
5017        .await?;
5018        Ok(())
5019    }
5020
5021    async fn start_deadline_finalization(
5022        &self,
5023        thread_id: &ThreadId,
5024        turn_id: &TurnId,
5025        transcript: &mut Vec<TranscriptItem>,
5026        remaining_seconds: u64,
5027    ) -> anyhow::Result<()> {
5028        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5029            remaining_seconds,
5030        ));
5031        self.persist_turn_item(thread_id, turn_id, &item).await?;
5032        transcript.push(item);
5033        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5034            thread_id: thread_id.clone(),
5035            turn_id: turn_id.clone(),
5036            summary: turn_partial_result(transcript),
5037            timestamp: OffsetDateTime::now_utc(),
5038        }))
5039        .await;
5040        Ok(())
5041    }
5042
5043    /// Emits the reliability-limit event for a continuation (loop persistence)
5044    /// without failing the turn. The caller resets the failure counter and
5045    /// injects a nudge so the round loop keeps going.
5046    async fn record_reliability_limit_continuation(
5047        &self,
5048        thread_id: &ThreadId,
5049        turn_id: &TurnId,
5050        provider: &str,
5051        model: &str,
5052        limit: ReliabilityLimitHit,
5053    ) {
5054        self.emit(RoderEvent::ReliabilityLimitRecorded(
5055            ReliabilityLimitRecorded {
5056                context: ReliabilityContext {
5057                    thread_id: thread_id.clone(),
5058                    turn_id: turn_id.clone(),
5059                    tool_id: None,
5060                    tool_name: None,
5061                    provider: Some(provider.to_string()),
5062                    model: Some(model.to_string()),
5063                },
5064                error_class: limit.error_class,
5065                limit_kind: limit.limit_kind,
5066                decision: limit.decision,
5067                current: limit.current,
5068                limit: limit.limit,
5069                details: ReliabilityDetails::redacted(&limit.message),
5070                timestamp: OffsetDateTime::now_utc(),
5071            },
5072        ))
5073        .await;
5074    }
5075
5076    async fn fail_turn_due_to_reliability_limit(
5077        &self,
5078        thread_id: &ThreadId,
5079        turn_id: &TurnId,
5080        provider: &str,
5081        model: &str,
5082        limit: ReliabilityLimitHit,
5083        transcript: &[TranscriptItem],
5084    ) -> anyhow::Result<()> {
5085        self.emit(RoderEvent::ReliabilityLimitRecorded(
5086            ReliabilityLimitRecorded {
5087                context: ReliabilityContext {
5088                    thread_id: thread_id.clone(),
5089                    turn_id: turn_id.clone(),
5090                    tool_id: None,
5091                    tool_name: None,
5092                    provider: Some(provider.to_string()),
5093                    model: Some(model.to_string()),
5094                },
5095                error_class: limit.error_class,
5096                limit_kind: limit.limit_kind,
5097                decision: limit.decision,
5098                current: limit.current,
5099                limit: limit.limit,
5100                details: ReliabilityDetails::redacted(&limit.message),
5101                timestamp: OffsetDateTime::now_utc(),
5102            },
5103        ))
5104        .await;
5105        let partial_result = turn_partial_result(transcript);
5106        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5107            thread_id: thread_id.clone(),
5108            turn_id: turn_id.clone(),
5109            summary: partial_result.clone(),
5110            timestamp: OffsetDateTime::now_utc(),
5111        }))
5112        .await;
5113        let message = format!("reliability limit reached: {}", limit.message);
5114        self.persist_turn_item(
5115            thread_id,
5116            turn_id,
5117            &TranscriptItem::Error(ErrorRecord {
5118                message: format!("{message}: {partial_result}"),
5119            }),
5120        )
5121        .await?;
5122        self.emit(RoderEvent::TurnFailed(TurnFailed {
5123            thread_id: thread_id.clone(),
5124            turn_id: turn_id.clone(),
5125            error: message.clone(),
5126            error_kind: Some("reliability_limit".to_string()),
5127            usage: None,
5128            timestamp: OffsetDateTime::now_utc(),
5129        }))
5130        .await;
5131        self.complete_team_member_turn_with_result(
5132            thread_id,
5133            turn_id,
5134            TeamMemberStatus::Failed,
5135            None,
5136            Some(format!("{message}: {partial_result}")),
5137        )
5138        .await?;
5139        Ok(())
5140    }
5141
5142    async fn append_steers(
5143        &self,
5144        req: &StartTurnRequest,
5145        turn_id: &TurnId,
5146        transcript: &mut Vec<TranscriptItem>,
5147        steers: Vec<QueuedTurnSteer>,
5148    ) -> anyhow::Result<()> {
5149        for queued in steers {
5150            let mut steer = queued.message;
5151            steer.text = steer.text.trim().to_string();
5152            if steer.text.is_empty() && steer.images.is_empty() {
5153                continue;
5154            }
5155            let item = TranscriptItem::UserMessage(steer);
5156            self.persist_turn_item(&req.thread_id, turn_id, &item)
5157                .await?;
5158            transcript.push(item);
5159            if let Some(ack) = queued.mailbox_ack {
5160                self.teams
5161                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5162                    .await?;
5163            }
5164        }
5165        Ok(())
5166    }
5167
5168    async fn persist_model_profile_segment(
5169        &self,
5170        thread_id: &ThreadId,
5171        turn_id: &TurnId,
5172        profile: Option<&ModelHarnessProfile>,
5173        provider: &str,
5174        model: &str,
5175        segment: &str,
5176    ) -> anyhow::Result<()> {
5177        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5178            profile, provider, model, segment,
5179        ));
5180        self.persist_turn_item(thread_id, turn_id, &item).await
5181    }
5182
5183    /**
5184     * Allowlists apply to built-in tools only; external tools are advertised with their
5185     * host-supplied schemas as given. An external tool shadows a built-in with the same name in
5186     * both advertisement and dispatch (see `route_tool_call`).
5187     */
5188    fn filtered_tool_specs(
5189        &self,
5190        cfg: &RuntimeConfig,
5191        model: &str,
5192        profile: Option<&ModelHarnessProfile>,
5193        thread_allowlist: &[String],
5194        external_tools: &[roder_api::tools::ToolSpec],
5195    ) -> Vec<roder_api::tools::ToolSpec> {
5196        let mut specs = self
5197            .tool_registry
5198            .specs_for_edit_tool_with_schema_policy(
5199                edit_tool_for_model(cfg, model),
5200                schema_policy_for_model(profile),
5201            )
5202            .into_iter()
5203            .filter(|spec| {
5204                allowlist_permits(&cfg.tool_allowlist, &spec.name)
5205                    && allowlist_permits(thread_allowlist, &spec.name)
5206                    && !external_tools.iter().any(|tool| tool.name == spec.name)
5207            })
5208            .collect::<Vec<_>>();
5209        specs.extend(external_tools.iter().cloned());
5210        specs
5211    }
5212
5213    fn task_ledger_tool_specs(
5214        &self,
5215        profile: Option<&ModelHarnessProfile>,
5216    ) -> Vec<roder_api::tools::ToolSpec> {
5217        self.tool_registry
5218            .get(TASK_LEDGER_TOOL_NAME)
5219            .map(|tool| {
5220                tool.spec()
5221                    .normalized_for_model_profile(schema_policy_for_model(profile))
5222            })
5223            .into_iter()
5224            .collect()
5225    }
5226
5227    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5228        self.registry
5229            .inference_engine(provider)
5230            .or_else(|| {
5231                self.registry
5232                    .default_inference_engine()
5233                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5234            })
5235            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5236    }
5237
5238    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5239        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5240            let _ = self.persist_turn_lifecycle_record(&record).await;
5241        }
5242        let envelope = self.bus.emit(event);
5243        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5244            && should_persist_thread_event(thread_id)
5245        {
5246            let _ = store.append_event(thread_id, &envelope).await;
5247        }
5248        // Registered event sinks (e.g. process extensions) receive the
5249        // persisted envelope through bounded per-sink queues; a slow sink
5250        // never blocks emit or turn progress.
5251        let dispatcher = self
5252            .event_sink_dispatcher
5253            .get_or_init(|| async {
5254                crate::event_sink_dispatch::EventSinkDispatcher::start(
5255                    &self.registry.event_sinks,
5256                    self.bus.clone(),
5257                )
5258            })
5259            .await;
5260        if !dispatcher.is_empty() {
5261            dispatcher.dispatch(&envelope, &self.bus);
5262        }
5263        envelope
5264    }
5265
5266    /// Records a projected thread item event and persists it through the configured thread store.
5267    ///
5268    /// App-server protocol notification bridges currently call this after translating runtime
5269    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5270    /// must make the same call if they need the derived item event stream persisted.
5271    pub async fn record_thread_item_event_kind(
5272        &self,
5273        thread_id: &ThreadId,
5274        turn_id: &TurnId,
5275        timestamp: OffsetDateTime,
5276        kind: ThreadItemEventKind,
5277    ) -> anyhow::Result<ThreadItemEvent> {
5278        let seq = self.next_thread_item_event_seq(thread_id).await?;
5279        let item_event = ThreadItemEvent {
5280            seq,
5281            event_id: format!("{turn_id}-item-event-{seq}"),
5282            thread_id: thread_id.clone(),
5283            turn_id: turn_id.clone(),
5284            timestamp,
5285            event: kind,
5286        };
5287        if let Some(store) = &self.thread_store {
5288            store.append_item_event(thread_id, &item_event).await?;
5289        }
5290        self.remember_thread_item_event(&item_event).await?;
5291        Ok(item_event)
5292    }
5293
5294    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5295        self.ensure_thread_item_cache(thread_id).await?;
5296        Ok(self
5297            .thread_item_cache
5298            .lock()
5299            .await
5300            .next_item_event_seq(thread_id))
5301    }
5302
5303    pub async fn thread_item_exists(
5304        &self,
5305        thread_id: &ThreadId,
5306        turn_id: &TurnId,
5307        item_id: &str,
5308    ) -> anyhow::Result<bool> {
5309        self.ensure_thread_item_cache(thread_id).await?;
5310        Ok(self
5311            .thread_item_cache
5312            .lock()
5313            .await
5314            .thread_item_exists(thread_id, turn_id, item_id))
5315    }
5316
5317    pub async fn current_reasoning_item_id(
5318        &self,
5319        thread_id: &ThreadId,
5320        turn_id: &TurnId,
5321    ) -> anyhow::Result<Option<String>> {
5322        self.ensure_thread_item_cache(thread_id).await?;
5323        Ok(self
5324            .thread_item_cache
5325            .lock()
5326            .await
5327            .current_reasoning_item_id(thread_id, turn_id))
5328    }
5329
5330    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5331        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5332        self.thread_item_cache
5333            .lock()
5334            .await
5335            .remember_item_event(item_event);
5336        Ok(())
5337    }
5338
5339    pub async fn latest_transcript_item_index(
5340        &self,
5341        thread_id: &ThreadId,
5342        turn_id: &TurnId,
5343    ) -> anyhow::Result<Option<usize>> {
5344        self.ensure_thread_item_cache(thread_id).await?;
5345        Ok(self
5346            .thread_item_cache
5347            .lock()
5348            .await
5349            .latest_transcript_item_index(thread_id, turn_id))
5350    }
5351
5352    async fn next_transcript_item_index(
5353        &self,
5354        thread_id: &ThreadId,
5355        turn_id: &TurnId,
5356    ) -> anyhow::Result<usize> {
5357        self.ensure_thread_item_cache(thread_id).await?;
5358        Ok(self
5359            .thread_item_cache
5360            .lock()
5361            .await
5362            .next_transcript_item_index(thread_id, turn_id))
5363    }
5364
5365    async fn remember_transcript_item_index(
5366        &self,
5367        thread_id: &ThreadId,
5368        turn_id: &TurnId,
5369        item_index: usize,
5370    ) -> anyhow::Result<()> {
5371        self.ensure_thread_item_cache(thread_id).await?;
5372        self.thread_item_cache
5373            .lock()
5374            .await
5375            .remember_transcript_item_index(thread_id, turn_id, item_index);
5376        Ok(())
5377    }
5378
5379    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5380        if self
5381            .thread_item_cache
5382            .lock()
5383            .await
5384            .contains_thread(thread_id)
5385        {
5386            return Ok(());
5387        }
5388
5389        let snapshot = if let Some(store) = &self.thread_store {
5390            store.load_thread(thread_id).await?
5391        } else {
5392            None
5393        };
5394        self.thread_item_cache.lock().await.ensure_thread(
5395            thread_id,
5396            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5397        );
5398        Ok(())
5399    }
5400
5401    pub(crate) async fn persist_turn_item(
5402        &self,
5403        thread_id: &ThreadId,
5404        turn_id: &TurnId,
5405        item: &TranscriptItem,
5406    ) -> anyhow::Result<()> {
5407        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5408        let timestamp = OffsetDateTime::now_utc();
5409        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5410            thread_id: thread_id.clone(),
5411            turn_id: turn_id.clone(),
5412            item_type: match item {
5413                TranscriptItem::UserMessage(_) => "user_message",
5414                TranscriptItem::AssistantMessage(_) => "assistant_message",
5415                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5416                TranscriptItem::ToolCall(_) => "tool_call",
5417                TranscriptItem::ToolResult(_) => "tool_result",
5418                TranscriptItem::FileChange(_) => "file_change",
5419                TranscriptItem::ContextCompaction(_) => "context_compaction",
5420                TranscriptItem::Error(_) => "error",
5421                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5422            }
5423            .to_string(),
5424            item_index: Some(item_index),
5425            item: Some(item.clone()),
5426            timestamp,
5427        }))
5428        .await;
5429        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5430            .await?;
5431        Ok(())
5432    }
5433}
5434
5435fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5436    transcript.iter().any(|item| {
5437        matches!(
5438            item,
5439            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5440        )
5441    })
5442}
5443
5444fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5445    transcript.iter().any(|item| {
5446        matches!(
5447            item,
5448            TranscriptItem::ToolResult(result)
5449                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5450        )
5451    })
5452}
5453
5454fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5455    let latest = transcript.iter().rev().find_map(|item| match item {
5456        TranscriptItem::ToolResult(result)
5457            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5458        {
5459            Some(result.result.as_str())
5460        }
5461        _ => None,
5462    })?;
5463    if !task_ledger_has_open_items(latest) {
5464        return None;
5465    }
5466
5467    let mut ledger = latest.chars().take(1500).collect::<String>();
5468    if latest.chars().nth(1500).is_some() {
5469        ledger.push_str("...");
5470    }
5471    Some(format!(
5472        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5473    ))
5474}
5475
5476fn task_ledger_deadline_completion_prompt(
5477    remaining_seconds: u64,
5478    reserve_seconds: u64,
5479    completion_prompt: &str,
5480) -> String {
5481    format!(
5482        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5483    )
5484}
5485
5486fn task_ledger_scoreable_checkpoint_prompt(
5487    remaining_seconds: u64,
5488    completion_prompt: &str,
5489) -> String {
5490    format!(
5491        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5492    )
5493}
5494
5495fn task_ledger_has_open_items(ledger: &str) -> bool {
5496    ledger.lines().any(|line| {
5497        let line = line.trim_start();
5498        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5499    })
5500}
5501
5502fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5503    if !cfg.runtime_profile.is_non_interactive() {
5504        return None;
5505    }
5506    cfg.turn_deadline_seconds
5507        .filter(|seconds| *seconds > 0)
5508        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5509}
5510
5511fn deadline_expired(deadline: OffsetDateTime) -> bool {
5512    OffsetDateTime::now_utc() >= deadline
5513}
5514
5515pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5516    let deadline = deadline?;
5517    if deadline <= OffsetDateTime::now_utc() {
5518        return Some(0);
5519    }
5520    Some(
5521        (deadline - OffsetDateTime::now_utc())
5522            .unsigned_abs()
5523            .as_secs()
5524            .max(1),
5525    )
5526}
5527
5528fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5529    let now = OffsetDateTime::now_utc();
5530    if deadline <= now {
5531        return tokio::time::Instant::now();
5532    }
5533    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5534}
5535
5536fn inference_timeout_deadline(
5537    deadline: Option<OffsetDateTime>,
5538    runtime_profile: RuntimeProfile,
5539    task_ledger_required: bool,
5540    reserve_seconds: u64,
5541    finalization_requested: bool,
5542    task_ledger_scoreable_checkpoints: u8,
5543    transcript: &[TranscriptItem],
5544) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5545    let deadline = deadline?;
5546    if runtime_profile == RuntimeProfile::Eval
5547        && task_ledger_required
5548        && !finalization_requested
5549        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5550        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5551        && task_ledger_completion_prompt(transcript).is_some()
5552    {
5553        let checkpoint_deadline =
5554            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5555        if checkpoint_deadline > OffsetDateTime::now_utc() {
5556            return Some((
5557                checkpoint_deadline,
5558                InferenceTimeoutAction::ScoreableCheckpoint,
5559            ));
5560        }
5561    }
5562    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5563        return Some((
5564            deadline - Duration::seconds(reserve_seconds as i64),
5565            InferenceTimeoutAction::Finalization,
5566        ));
5567    }
5568    Some((deadline, InferenceTimeoutAction::Finalization))
5569}
5570
5571fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5572    let tool_results = transcript
5573        .iter()
5574        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5575        .count();
5576    let assistant_messages = transcript
5577        .iter()
5578        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5579        .count();
5580    format!(
5581        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5582        transcript.len()
5583    )
5584}
5585
5586fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5587    let level = effective_reasoning_for_model(cfg, model);
5588    match level.as_str() {
5589        "" | REASONING_NONE => ReasoningConfig::default(),
5590        level => ReasoningConfig {
5591            enabled: true,
5592            level: Some(level.to_string()),
5593        },
5594    }
5595}
5596
5597fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5598    let entry = lookup_model(model)?;
5599    if !entry.supports_compaction {
5600        return None;
5601    }
5602    cfg.auto_compact_token_limit
5603        .or_else(|| {
5604            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5605        })
5606        .or(Some(entry.auto_compact_token_limit))
5607        .filter(|threshold| *threshold > 0)
5608}
5609
5610pub(crate) fn tool_search_for_provider_model(
5611    cfg: &RuntimeConfig,
5612    provider: &str,
5613    model: &str,
5614) -> ToolSearchConfig {
5615    let mut resolved = cfg.tool_search.clone();
5616    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5617        provider_config.apply_to(&mut resolved);
5618    }
5619    if let Some(model_config) = cfg.model_tool_search.get(model) {
5620        model_config.apply_to(&mut resolved);
5621    }
5622    resolved
5623}
5624
5625fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5626    cfg.model_parallel_tool_calls
5627        .get(model)
5628        .copied()
5629        .or_else(|| {
5630            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5631        })
5632        .unwrap_or(true)
5633}
5634
5635/// Count the children an `agent_swarm` call will launch from its arguments
5636/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5637fn agent_swarm_child_count(arguments: &str) -> usize {
5638    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5639        .map(|request| request.items.len() + request.resume_agent_ids.len())
5640        .unwrap_or(0)
5641}
5642
5643/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5644/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5645/// buckets default to 0. Returns `None` when the text is not a swarm result.
5646fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5647    if !text.contains("<agent_swarm_result>") {
5648        return None;
5649    }
5650    let bucket = |label: &str| -> usize {
5651        let needle = format!("{label}: ");
5652        text.find(&needle)
5653            .map(|start| start + needle.len())
5654            .map(|start| {
5655                text[start..]
5656                    .chars()
5657                    .take_while(|c| c.is_ascii_digit())
5658                    .collect::<String>()
5659            })
5660            .and_then(|digits| digits.parse().ok())
5661            .unwrap_or(0)
5662    };
5663    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5664}
5665
5666fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5667    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5668    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5669        return ultracode_reasoning_level_for_model(
5670            model,
5671            &cfg.speed_policy.ultracode_reasoning,
5672            &base_reasoning,
5673        );
5674    }
5675    base_reasoning
5676}
5677
5678fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5679    let Some(entry) = lookup_model(model) else {
5680        return cfg
5681            .reasoning
5682            .clone()
5683            .unwrap_or_else(|| REASONING_NONE.to_string());
5684    };
5685    if entry.supported_reasoning.is_empty() {
5686        return REASONING_NONE.to_string();
5687    }
5688    cfg.reasoning
5689        .as_deref()
5690        .filter(|reasoning| {
5691            entry
5692                .supported_reasoning
5693                .iter()
5694                .any(|option| option.effort == *reasoning)
5695        })
5696        .map(str::to_string)
5697        .or_else(|| {
5698            model_profile_for_model(cfg, model)
5699                .and_then(|profile| profile.reasoning.orientation)
5700                .filter(|reasoning| {
5701                    entry
5702                        .supported_reasoning
5703                        .iter()
5704                        .any(|option| option.effort == reasoning)
5705                })
5706        })
5707        .unwrap_or_else(|| entry.default_reasoning.to_string())
5708}
5709
5710fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5711    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5712        return Ok(());
5713    }
5714    let Some(entry) = lookup_model(model) else {
5715        return Ok(());
5716    };
5717    if entry
5718        .supported_reasoning
5719        .iter()
5720        .any(|option| option.effort == effort)
5721    {
5722        Ok(())
5723    } else {
5724        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5725    }
5726}
5727
5728fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5729    let Some(reasoning) = cfg.reasoning.as_deref() else {
5730        return Ok(());
5731    };
5732    let Some(entry) = lookup_model(&cfg.default_model) else {
5733        return Ok(());
5734    };
5735    if entry.provider != PROVIDER_GEMINI {
5736        return Ok(());
5737    }
5738    validate_reasoning_effort(&cfg.default_model, reasoning)
5739}
5740
5741fn validate_runtime_inference_router_config(
5742    registry: &ExtensionRegistry,
5743    cfg: &RuntimeConfig,
5744) -> anyhow::Result<()> {
5745    if !cfg.inference_router.enabled {
5746        return Ok(());
5747    }
5748    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5749        anyhow::bail!("inference_router.enabled requires inference_router.router");
5750    };
5751    if registry.inference_router(router_id).is_some() {
5752        return Ok(());
5753    }
5754    let available = registry
5755        .inference_routers
5756        .iter()
5757        .map(|router| router.id())
5758        .collect::<Vec<_>>()
5759        .join(", ");
5760    if available.is_empty() {
5761        anyhow::bail!("inference router {router_id:?} is not registered");
5762    }
5763    anyhow::bail!(
5764        "inference router {router_id:?} is not registered; available routers: {available}"
5765    );
5766}
5767
5768fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5769    lookup_model(model)
5770        .map(|entry| {
5771            entry
5772                .supported_reasoning
5773                .iter()
5774                .any(|option| option.effort == effort)
5775        })
5776        .unwrap_or(false)
5777}
5778
5779fn is_final_answer_phase(phase: Option<&str>) -> bool {
5780    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5781}
5782
5783fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5784    cfg.model_edit_tools
5785        .get(model)
5786        .map(String::as_str)
5787        .or_else(|| {
5788            cfg.model_profiles
5789                .get(model)
5790                .and_then(|profile| profile.edit_tool.as_deref())
5791        })
5792        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5793        .or(Some(EDIT_TOOL_EDIT))
5794}
5795
5796fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5797    cfg.model_profiles
5798        .get(model)
5799        .cloned()
5800        .or_else(|| built_in_model_profile(model))
5801}
5802
5803pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5804    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5805}
5806
5807/// Provider-aware profile resolution for the active turn.
5808///
5809/// Many model ids are shared across providers (for example Cursor proxies
5810/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5811/// first catalog entry, which assigns cross-provider ids the wrong family and
5812/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5813/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5814/// catalog entry, keeping user-configured profile overrides as the top
5815/// precedence.
5816fn model_profile_for_provider_model(
5817    cfg: &RuntimeConfig,
5818    provider: &str,
5819    model: &str,
5820) -> Option<ModelHarnessProfile> {
5821    cfg.model_profiles
5822        .get(model)
5823        .cloned()
5824        .or_else(|| built_in_model_profile_for_provider(provider, model))
5825}
5826
5827fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5828    profile
5829        .map(|profile| profile.schema_policy)
5830        .unwrap_or_default()
5831}
5832
5833fn model_profile_segment_metadata(
5834    profile: Option<&ModelHarnessProfile>,
5835    provider: &str,
5836    model: &str,
5837    segment: &str,
5838) -> serde_json::Value {
5839    serde_json::json!({
5840        "kind": MODEL_PROFILE_TRACE_KIND,
5841        "segment": segment,
5842        "provider": provider,
5843        "model": model,
5844        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5845        "providerFamily": profile.map(|profile| profile.provider_family),
5846        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5847        "schemaPolicy": profile.map(|profile| profile.schema_policy),
5848        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5849        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5850        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5851    })
5852}
5853
5854fn model_switch_summary(
5855    transcript: &[TranscriptItem],
5856    profile: Option<&ModelHarnessProfile>,
5857    provider: &str,
5858    model: &str,
5859    tools: &[roder_api::tools::ToolSpec],
5860) -> Option<String> {
5861    let previous = latest_model_profile_segment(transcript)?;
5862    let previous_model = previous
5863        .get("model")
5864        .and_then(serde_json::Value::as_str)
5865        .unwrap_or_default();
5866    let previous_provider = previous
5867        .get("provider")
5868        .and_then(serde_json::Value::as_str)
5869        .unwrap_or_default();
5870    if previous_model == model && previous_provider == provider {
5871        return None;
5872    }
5873
5874    let previous_profile = previous
5875        .get("profileModel")
5876        .and_then(serde_json::Value::as_str)
5877        .unwrap_or(previous_model);
5878    let current_profile = profile
5879        .map(|profile| profile.model.as_str())
5880        .unwrap_or(model);
5881    let previous_edit_tool = previous
5882        .get("editTool")
5883        .and_then(serde_json::Value::as_str)
5884        .unwrap_or("none");
5885    let current_edit_tool = profile
5886        .and_then(|profile| profile.edit_tool.as_deref())
5887        .unwrap_or("none");
5888    let tool_names = tools
5889        .iter()
5890        .map(|tool| tool.name.as_str())
5891        .take(12)
5892        .collect::<Vec<_>>()
5893        .join(", ");
5894    Some(format!(
5895        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
5896        if tool_names.is_empty() {
5897            "none"
5898        } else {
5899            &tool_names
5900        }
5901    ))
5902}
5903
5904fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
5905    transcript.iter().rev().find_map(|item| {
5906        let TranscriptItem::ProviderMetadata(value) = item else {
5907            return None;
5908        };
5909        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
5910            .then_some(value)
5911    })
5912}
5913
5914pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
5915    match value.trim() {
5916        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
5917        _ => anyhow::bail!(
5918            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
5919        ),
5920    }
5921}
5922
5923fn should_persist_thread_event(thread_id: &str) -> bool {
5924    !is_synthetic_event_thread_id(thread_id)
5925}
5926
5927#[cfg(test)]
5928#[path = "runtime/codex_v2_tests.rs"]
5929mod codex_v2_tests;
5930
5931#[cfg(test)]
5932#[path = "runtime/codex_v2_lifecycle_tests.rs"]
5933mod codex_v2_lifecycle_tests;
5934
5935#[cfg(test)]
5936mod tests {
5937    use super::*;
5938    use futures::stream;
5939    use roder_api::catalog::{
5940        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
5941        REASONING_NONE, REASONING_XHIGH,
5942    };
5943    use roder_api::extension::ExtensionRegistryBuilder;
5944    use roder_api::inference::{
5945        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
5946        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
5947        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
5948        ReasoningEffortDescriptor,
5949    };
5950    use roder_api::inference_routing::{
5951        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
5952    };
5953    use roder_api::thread::ThreadStoreFactory;
5954    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
5955    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
5956    use std::sync::Mutex as StdMutex;
5957
5958    fn test_workspace() -> String {
5959        std::env::current_dir().unwrap().display().to_string()
5960    }
5961
5962    struct MetadataMissingStore;
5963
5964    #[async_trait::async_trait]
5965    impl ThreadStore for MetadataMissingStore {
5966        fn id(&self) -> roder_api::thread::ThreadStoreId {
5967            "metadata-missing-store".to_string()
5968        }
5969
5970        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
5971            Ok(metadata)
5972        }
5973
5974        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
5975            Ok(Vec::new())
5976        }
5977
5978        async fn load_thread(
5979            &self,
5980            _thread_id: &ThreadId,
5981        ) -> anyhow::Result<Option<ThreadSnapshot>> {
5982            Ok(Some(ThreadSnapshot {
5983                metadata: None,
5984                ..ThreadSnapshot::default()
5985            }))
5986        }
5987
5988        async fn append_event(
5989            &self,
5990            _thread_id: &ThreadId,
5991            _envelope: &EventEnvelope,
5992        ) -> anyhow::Result<()> {
5993            Ok(())
5994        }
5995    }
5996
5997    struct MetadataMissingStoreFactory;
5998
5999    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6000        fn id(&self) -> roder_api::thread::ThreadStoreId {
6001            "metadata-missing-store".to_string()
6002        }
6003
6004        fn create(&self) -> Arc<dyn ThreadStore> {
6005            Arc::new(MetadataMissingStore)
6006        }
6007    }
6008
6009    #[test]
6010    fn synthetic_app_server_events_are_not_thread_events() {
6011        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6012            assert!(!should_persist_thread_event(thread_id));
6013        }
6014        assert!(should_persist_thread_event("thread-discovery"));
6015        assert!(should_persist_thread_event("thread-plan"));
6016        assert!(should_persist_thread_event("thread-process"));
6017        assert!(should_persist_thread_event("thread-1"));
6018    }
6019
6020    #[test]
6021    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6022        assert_eq!(
6023            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6024            Some(945_000)
6025        );
6026        assert_eq!(
6027            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6028            Some(115_200)
6029        );
6030    }
6031
6032    #[test]
6033    fn server_side_compaction_respects_explicit_config_override() {
6034        let cfg = RuntimeConfig {
6035            auto_compact_token_limit: Some(123_456),
6036            ..RuntimeConfig::default()
6037        };
6038
6039        assert_eq!(
6040            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6041            Some(123_456)
6042        );
6043    }
6044
6045    #[tokio::test]
6046    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6047        let captured = Arc::new(StdMutex::new(None));
6048        let mut builder = ExtensionRegistryBuilder::new();
6049        builder.inference_engine(Arc::new(CapturingEngine {
6050            request: captured.clone(),
6051        }));
6052        let thread_root = std::env::temp_dir().join(format!(
6053            "roder-pre-request-compaction-{}",
6054            uuid::Uuid::new_v4()
6055        ));
6056        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6057            base_path: thread_root.clone(),
6058        }));
6059        let runtime = Arc::new(
6060            Runtime::new(
6061                builder.build().unwrap(),
6062                RuntimeConfig {
6063                    default_provider: PROVIDER_MOCK.to_string(),
6064                    default_model: "gpt-5.5".to_string(),
6065                    file_backed_dynamic_context: true,
6066                    ..RuntimeConfig::default()
6067                },
6068            )
6069            .unwrap(),
6070        );
6071        let thread_id = runtime
6072            .create_thread(Some("Pre-request compaction".to_string()))
6073            .await
6074            .unwrap()
6075            .thread_id;
6076        let old_turn = "old-turn".to_string();
6077        runtime
6078            .persist_turn_item(
6079                &thread_id,
6080                &old_turn,
6081                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6082            )
6083            .await
6084            .unwrap();
6085
6086        let mut events = runtime.subscribe_events();
6087        runtime
6088            .start_turn(StartTurnRequest {
6089                thread_id: thread_id.clone(),
6090                message: "continue".to_string(),
6091                images: Vec::new(),
6092                provider_override: None,
6093                model_override: None,
6094                reasoning_override: None,
6095                workspace: test_workspace(),
6096                instructions: InstructionBundle::default(),
6097                developer_context: None,
6098                task_ledger_required: false,
6099            })
6100            .await
6101            .unwrap();
6102        loop {
6103            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6104                .await
6105                .unwrap()
6106                .unwrap();
6107            if envelope.thread_id.as_deref() == Some(&thread_id)
6108                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6109            {
6110                break;
6111            }
6112        }
6113
6114        let request = captured.lock().unwrap().clone().unwrap();
6115        assert!(
6116            matches!(
6117                request.transcript.first(),
6118                Some(TranscriptItem::ContextCompaction(_))
6119            ),
6120            "provider request should start with a local emergency compaction item"
6121        );
6122        assert!(
6123            request.transcript.len() < 4,
6124            "provider request should not replay the full oversized prior transcript: {:?}",
6125            request.transcript
6126        );
6127
6128        let _ = std::fs::remove_dir_all(thread_root);
6129    }
6130
6131    #[tokio::test]
6132    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6133        let captured = Arc::new(StdMutex::new(None));
6134        let mut builder = ExtensionRegistryBuilder::new();
6135        builder.inference_engine(Arc::new(CapturingEngine {
6136            request: captured.clone(),
6137        }));
6138        let thread_root = std::env::temp_dir().join(format!(
6139            "roder-context-failure-continue-{}",
6140            uuid::Uuid::new_v4()
6141        ));
6142        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6143            base_path: thread_root.clone(),
6144        }));
6145        let runtime = Arc::new(
6146            Runtime::new(
6147                builder.build().unwrap(),
6148                RuntimeConfig {
6149                    default_provider: PROVIDER_MOCK.to_string(),
6150                    default_model: "gpt-5.5".to_string(),
6151                    file_backed_dynamic_context: true,
6152                    ..RuntimeConfig::default()
6153                },
6154            )
6155            .unwrap(),
6156        );
6157        let thread_id = runtime
6158            .create_thread(Some("Context failure continue".to_string()))
6159            .await
6160            .unwrap()
6161            .thread_id;
6162        let failed_turn = "failed-turn".to_string();
6163        runtime
6164            .persist_turn_item(
6165                &thread_id,
6166                &failed_turn,
6167                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6168            )
6169            .await
6170            .unwrap();
6171        runtime
6172            .persist_turn_item(
6173                &thread_id,
6174                &failed_turn,
6175                &TranscriptItem::Error(ErrorRecord {
6176                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6177                        .to_string(),
6178                }),
6179            )
6180            .await
6181            .unwrap();
6182
6183        let mut events = runtime.subscribe_events();
6184        runtime
6185            .start_turn(StartTurnRequest {
6186                thread_id: thread_id.clone(),
6187                message: "continue".to_string(),
6188                images: Vec::new(),
6189                provider_override: None,
6190                model_override: None,
6191                reasoning_override: None,
6192                workspace: test_workspace(),
6193                instructions: InstructionBundle::default(),
6194                developer_context: None,
6195                task_ledger_required: false,
6196            })
6197            .await
6198            .unwrap();
6199        loop {
6200            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6201                .await
6202                .unwrap()
6203                .unwrap();
6204            if envelope.thread_id.as_deref() == Some(&thread_id)
6205                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6206            {
6207                break;
6208            }
6209        }
6210
6211        let request = captured.lock().unwrap().clone().unwrap();
6212        assert!(
6213            matches!(
6214                request.transcript.first(),
6215                Some(TranscriptItem::ContextCompaction(_))
6216            ),
6217            "provider request after context-window failure should start with local compaction"
6218        );
6219        assert!(
6220            request
6221                .transcript
6222                .iter()
6223                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6224            "current continue prompt must be preserved: {:?}",
6225            request.transcript
6226        );
6227        assert!(
6228            !request.transcript.iter().any(
6229                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6230            ),
6231            "raw prior context-window error should be summarized, not replayed: {:?}",
6232            request.transcript
6233        );
6234
6235        let _ = std::fs::remove_dir_all(thread_root);
6236    }
6237
6238    #[tokio::test]
6239    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6240        let workspace = test_workspace();
6241        let mut builder = ExtensionRegistryBuilder::new();
6242        builder.inference_engine(Arc::new(FakeInferenceEngine));
6243        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6244        let runtime = Runtime::new(
6245            builder.build().unwrap(),
6246            RuntimeConfig {
6247                workspace: Some(workspace.clone()),
6248                ..RuntimeConfig::default()
6249            },
6250        )
6251        .unwrap();
6252
6253        let resolved = runtime
6254            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6255            .await
6256            .unwrap();
6257
6258        assert_eq!(resolved, workspace);
6259    }
6260
6261    #[tokio::test]
6262    async fn automations_can_create_project_thread_with_model_overrides() {
6263        let runtime = Runtime::fake().unwrap();
6264        let workspace = std::env::temp_dir().join("project");
6265        let metadata = runtime
6266            .create_thread_with(CreateThreadRequest {
6267                title: Some("Automation: nightly status".to_string()),
6268                workspace: workspace.display().to_string(),
6269                workspace_id: None,
6270                root_id: None,
6271                provider: Some("mock".to_string()),
6272                model: Some("mock".to_string()),
6273                selection_mode: None,
6274                tool_allowlist: Vec::new(),
6275                developer_instructions: None,
6276                external_tools: Vec::new(),
6277                runner: None,
6278            })
6279            .await
6280            .unwrap();
6281
6282        assert_eq!(
6283            metadata.title.as_deref(),
6284            Some("Automation: nightly status")
6285        );
6286        assert_eq!(metadata.workspace, workspace.display().to_string());
6287        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6288        assert_eq!(metadata.model.as_deref(), Some("mock"));
6289    }
6290
6291    #[tokio::test]
6292    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6293        let captured = Arc::new(StdMutex::new(None));
6294        let mut builder = ExtensionRegistryBuilder::new();
6295        builder.inference_engine(Arc::new(CapturingEngine {
6296            request: captured.clone(),
6297        }));
6298        let thread_root = std::env::temp_dir().join(format!(
6299            "roder-provider-compaction-boundary-{}",
6300            uuid::Uuid::new_v4()
6301        ));
6302        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6303            base_path: thread_root.clone(),
6304        }));
6305        let runtime = Arc::new(
6306            Runtime::new(
6307                builder.build().unwrap(),
6308                RuntimeConfig {
6309                    default_provider: PROVIDER_MOCK.to_string(),
6310                    default_model: "gpt-5.5".to_string(),
6311                    ..RuntimeConfig::default()
6312                },
6313            )
6314            .unwrap(),
6315        );
6316        let thread_id = runtime
6317            .create_thread(Some("Provider compaction boundary".to_string()))
6318            .await
6319            .unwrap()
6320            .thread_id;
6321        let old_turn = "old-turn".to_string();
6322        runtime
6323            .persist_turn_item(
6324                &thread_id,
6325                &old_turn,
6326                &TranscriptItem::UserMessage(UserMessage::text(
6327                    "old history that must not be replayed after provider compaction",
6328                )),
6329            )
6330            .await
6331            .unwrap();
6332        runtime
6333            .persist_turn_item(
6334                &thread_id,
6335                &old_turn,
6336                &TranscriptItem::AssistantMessage(AssistantMessage {
6337                    text: "old answer".to_string(),
6338                    phase: None,
6339                }),
6340            )
6341            .await
6342            .unwrap();
6343        runtime
6344            .persist_turn_item(
6345                &thread_id,
6346                &old_turn,
6347                &TranscriptItem::ProviderMetadata(serde_json::json!({
6348                    "output": [{
6349                        "id": "cmp_1",
6350                        "type": "compaction",
6351                        "encrypted_content": "opaque-state"
6352                    }]
6353                })),
6354            )
6355            .await
6356            .unwrap();
6357        runtime
6358            .persist_turn_item(
6359                &thread_id,
6360                &old_turn,
6361                &TranscriptItem::AssistantMessage(AssistantMessage {
6362                    text: "after compact".to_string(),
6363                    phase: None,
6364                }),
6365            )
6366            .await
6367            .unwrap();
6368
6369        let mut events = runtime.subscribe_events();
6370        runtime
6371            .start_turn(StartTurnRequest {
6372                thread_id: thread_id.clone(),
6373                message: "continue".to_string(),
6374                images: Vec::new(),
6375                provider_override: None,
6376                model_override: None,
6377                reasoning_override: None,
6378                workspace: test_workspace(),
6379                instructions: InstructionBundle::default(),
6380                developer_context: None,
6381                task_ledger_required: false,
6382            })
6383            .await
6384            .unwrap();
6385        loop {
6386            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6387                .await
6388                .unwrap()
6389                .unwrap();
6390            if envelope.thread_id.as_deref() == Some(&thread_id)
6391                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6392            {
6393                break;
6394            }
6395        }
6396
6397        let request = captured.lock().unwrap().clone().unwrap();
6398        let compaction_idx = request.transcript.iter().position(|item| {
6399            matches!(
6400                item,
6401                TranscriptItem::ProviderMetadata(metadata)
6402                    if crate::compaction::provider_metadata_has_compaction(metadata)
6403            )
6404        });
6405        assert!(
6406            compaction_idx.is_some(),
6407            "next provider request should include the provider compaction item: {:?}",
6408            request.transcript
6409        );
6410        // Skill/context injectors may prepend non-history items, but no prior-turn
6411        // conversation may appear before the latest provider compaction boundary.
6412        let history_before_boundary =
6413            request
6414                .transcript
6415                .iter()
6416                .take(compaction_idx.unwrap())
6417                .any(|item| match item {
6418                    TranscriptItem::AssistantMessage(_)
6419                    | TranscriptItem::ToolCall(_)
6420                    | TranscriptItem::ToolResult(_) => true,
6421                    TranscriptItem::UserMessage(message) => {
6422                        !message.text.contains("<skills>") && message.text != "continue"
6423                    }
6424                    _ => false,
6425                });
6426        assert!(
6427            !history_before_boundary,
6428            "pre-provider-compaction conversation must not be replayed: {:?}",
6429            request.transcript
6430        );
6431        assert!(
6432            !request.transcript.iter().any(|item| {
6433                matches!(
6434                    item,
6435                    TranscriptItem::UserMessage(message)
6436                        if message.text.contains("old history that must not be replayed")
6437                )
6438            }),
6439            "pre-provider-compaction history must not be replayed: {:?}",
6440            request.transcript
6441        );
6442        assert!(
6443            request.transcript.iter().any(|item| {
6444                matches!(
6445                    item,
6446                    TranscriptItem::UserMessage(message) if message.text == "continue"
6447                )
6448            }),
6449            "current continue prompt must be preserved: {:?}",
6450            request.transcript
6451        );
6452        assert_eq!(
6453            request.runtime.auto_compact_token_limit,
6454            Some(945_000),
6455            "gpt-5.5 should keep server-side auto compaction configured"
6456        );
6457
6458        let _ = std::fs::remove_dir_all(thread_root);
6459    }
6460
6461    #[test]
6462    fn server_side_compaction_is_only_enabled_for_supported_models() {
6463        let cfg = RuntimeConfig {
6464            auto_compact_token_limit: Some(123_456),
6465            ..RuntimeConfig::default()
6466        };
6467
6468        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6469        assert_eq!(
6470            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6471            None
6472        );
6473    }
6474
6475    #[test]
6476    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6477        let cfg = RuntimeConfig {
6478            reasoning: Some(REASONING_HIGH.to_string()),
6479            ..RuntimeConfig::default()
6480        };
6481
6482        assert_eq!(
6483            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6484            REASONING_NONE
6485        );
6486        assert_eq!(
6487            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6488            ReasoningConfig::default()
6489        );
6490    }
6491
6492    #[test]
6493    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6494        let cfg = RuntimeConfig {
6495            reasoning: Some(REASONING_MINIMAL.to_string()),
6496            ..RuntimeConfig::default()
6497        };
6498
6499        assert_eq!(
6500            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6501            REASONING_MEDIUM
6502        );
6503    }
6504
6505    #[test]
6506    fn unsupported_configured_gemini_reasoning_is_rejected() {
6507        let mut builder = ExtensionRegistryBuilder::new();
6508        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6509
6510        let err = match Runtime::new(
6511            builder.build().unwrap(),
6512            RuntimeConfig {
6513                default_model: "gemini-3.5-flash".to_string(),
6514                reasoning: Some(REASONING_XHIGH.to_string()),
6515                ..RuntimeConfig::default()
6516            },
6517        ) {
6518            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6519            Err(err) => err,
6520        };
6521
6522        assert!(
6523            err.to_string()
6524                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6525        );
6526    }
6527
6528    #[tokio::test]
6529    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6530        let runtime = Runtime::new(
6531            Runtime::fake().unwrap().registry,
6532            RuntimeConfig {
6533                reasoning: Some(REASONING_HIGH.to_string()),
6534                ..RuntimeConfig::default()
6535            },
6536        )
6537        .unwrap();
6538
6539        let cfg = runtime
6540            .select_provider(
6541                roder_api::catalog::PROVIDER_MOCK.to_string(),
6542                Some("claude-haiku-4-5-20251001".to_string()),
6543                Some(REASONING_NONE.to_string()),
6544            )
6545            .await
6546            .unwrap();
6547
6548        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6549        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6550    }
6551
6552    #[tokio::test]
6553    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6554        let runtime = Runtime::new(
6555            Runtime::fake().unwrap().registry,
6556            RuntimeConfig {
6557                reasoning: Some(REASONING_HIGH.to_string()),
6558                ..RuntimeConfig::default()
6559            },
6560        )
6561        .unwrap();
6562
6563        let cfg = runtime
6564            .select_provider(
6565                roder_api::catalog::PROVIDER_MOCK.to_string(),
6566                Some("mock".to_string()),
6567                Some(REASONING_NONE.to_string()),
6568            )
6569            .await
6570            .unwrap();
6571
6572        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6573    }
6574
6575    #[test]
6576    fn parallel_tool_calls_default_on_with_model_override() {
6577        assert!(parallel_tool_calls_for_model(
6578            &RuntimeConfig::default(),
6579            "custom-model"
6580        ));
6581
6582        let cfg = RuntimeConfig {
6583            model_parallel_tool_calls: std::collections::HashMap::from([(
6584                "custom-model".to_string(),
6585                false,
6586            )]),
6587            ..RuntimeConfig::default()
6588        };
6589
6590        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6591        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6592    }
6593
6594    #[test]
6595    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6596        let cfg = RuntimeConfig {
6597            model_profiles: std::collections::HashMap::from([(
6598                "gpt-5.5".to_string(),
6599                test_model_profile("gpt-5.5"),
6600            )]),
6601            ..RuntimeConfig::default()
6602        };
6603
6604        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6605
6606        let cfg = RuntimeConfig {
6607            model_parallel_tool_calls: std::collections::HashMap::from([(
6608                "gpt-5.5".to_string(),
6609                true,
6610            )]),
6611            ..cfg
6612        };
6613
6614        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6615    }
6616
6617    struct CapturingEngine {
6618        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6619    }
6620
6621    #[async_trait::async_trait]
6622    impl InferenceEngine for CapturingEngine {
6623        fn id(&self) -> String {
6624            roder_api::catalog::PROVIDER_MOCK.to_string()
6625        }
6626
6627        fn capabilities(&self) -> InferenceCapabilities {
6628            InferenceCapabilities::coding_agent_default()
6629        }
6630
6631        async fn list_models(
6632            &self,
6633            _ctx: InferenceProviderContext<'_>,
6634        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6635            Ok(roder_api::catalog::models_for_provider(
6636                roder_api::catalog::PROVIDER_MOCK,
6637                true,
6638            ))
6639        }
6640
6641        async fn stream_turn(
6642            &self,
6643            _ctx: InferenceTurnContext<'_>,
6644            request: AgentInferenceRequest,
6645        ) -> anyhow::Result<InferenceEventStream> {
6646            *self.request.lock().unwrap() = Some(request);
6647            Ok(Box::pin(stream::iter(vec![
6648                Ok(InferenceEvent::MessageDelta(MessageDelta {
6649                    text: "done".to_string(),
6650                    phase: None,
6651                })),
6652                Ok(InferenceEvent::Completed(CompletionMetadata {
6653                    stop_reason: Some("stop".to_string()),
6654                    provider_response_id: None,
6655                })),
6656            ])))
6657        }
6658    }
6659
6660    struct RoutingCaptureEngine {
6661        id: &'static str,
6662        models: Vec<ModelDescriptor>,
6663        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6664    }
6665
6666    #[async_trait::async_trait]
6667    impl InferenceEngine for RoutingCaptureEngine {
6668        fn id(&self) -> String {
6669            self.id.to_string()
6670        }
6671
6672        fn capabilities(&self) -> InferenceCapabilities {
6673            InferenceCapabilities::coding_agent_default()
6674        }
6675
6676        async fn list_models(
6677            &self,
6678            _ctx: InferenceProviderContext<'_>,
6679        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6680            Ok(self.models.clone())
6681        }
6682
6683        async fn stream_turn(
6684            &self,
6685            _ctx: InferenceTurnContext<'_>,
6686            request: AgentInferenceRequest,
6687        ) -> anyhow::Result<InferenceEventStream> {
6688            self.requests.lock().unwrap().push(request);
6689            Ok(Box::pin(stream::iter(vec![
6690                Ok(InferenceEvent::MessageDelta(MessageDelta {
6691                    text: "routed".to_string(),
6692                    phase: None,
6693                })),
6694                Ok(InferenceEvent::Completed(CompletionMetadata {
6695                    stop_reason: Some("stop".to_string()),
6696                    provider_response_id: None,
6697                })),
6698            ])))
6699        }
6700    }
6701
6702    struct StaticRouter {
6703        id: &'static str,
6704        decision: InferenceRoutingDecision,
6705        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6706    }
6707
6708    #[async_trait::async_trait]
6709    impl InferenceRouter for StaticRouter {
6710        fn id(&self) -> String {
6711            self.id.to_string()
6712        }
6713
6714        async fn route(
6715            &self,
6716            context: InferenceRoutingContext,
6717        ) -> anyhow::Result<InferenceRoutingDecision> {
6718            self.contexts.lock().unwrap().push(context);
6719            Ok(self.decision.clone())
6720        }
6721    }
6722
6723    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6724        ModelDescriptor {
6725            id: id.to_string(),
6726            name: id.to_string(),
6727            context_window: Some(128_000),
6728            default_reasoning: supported_reasoning
6729                .first()
6730                .map(|effort| (*effort).to_string()),
6731            supported_reasoning: supported_reasoning
6732                .iter()
6733                .map(|effort| ReasoningEffortDescriptor {
6734                    effort: (*effort).to_string(),
6735                    description: format!("{effort} reasoning"),
6736                })
6737                .collect(),
6738        }
6739    }
6740
6741    struct TaskLedgerCompletionGateEngine {
6742        calls: StdMutex<u32>,
6743        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6744    }
6745
6746    #[async_trait::async_trait]
6747    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6748        fn id(&self) -> String {
6749            roder_api::catalog::PROVIDER_MOCK.to_string()
6750        }
6751
6752        fn capabilities(&self) -> InferenceCapabilities {
6753            InferenceCapabilities::coding_agent_default()
6754        }
6755
6756        async fn list_models(
6757            &self,
6758            _ctx: InferenceProviderContext<'_>,
6759        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6760            Ok(roder_api::catalog::models_for_provider(
6761                roder_api::catalog::PROVIDER_MOCK,
6762                true,
6763            ))
6764        }
6765
6766        async fn stream_turn(
6767            &self,
6768            _ctx: InferenceTurnContext<'_>,
6769            request: AgentInferenceRequest,
6770        ) -> anyhow::Result<InferenceEventStream> {
6771            self.requests.lock().unwrap().push(request);
6772            let mut calls = self.calls.lock().unwrap();
6773            *calls += 1;
6774            let events = match *calls {
6775                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6776                    id: "ledger-open".to_string(),
6777                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6778                    arguments: serde_json::json!({
6779                        "tasks": [
6780                            {
6781                                "id": "inspect",
6782                                "content": "Inspect local assets",
6783                                "status": "completed",
6784                                "evidence": "listed workspace"
6785                            },
6786                            {
6787                                "id": "write",
6788                                "content": "Write /app/result.txt",
6789                                "status": "pending"
6790                            }
6791                        ],
6792                        "requireCompletionEvidence": true
6793                    })
6794                    .to_string(),
6795                }))],
6796                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6797                    id: "ledger-complete".to_string(),
6798                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6799                    arguments: serde_json::json!({
6800                        "tasks": [
6801                            {
6802                                "id": "inspect",
6803                                "content": "Inspect local assets",
6804                                "status": "completed",
6805                                "evidence": "listed workspace"
6806                            },
6807                            {
6808                                "id": "write",
6809                                "content": "Write /app/result.txt",
6810                                "status": "completed",
6811                                "evidence": "wrote answer"
6812                            }
6813                        ],
6814                        "requireCompletionEvidence": true
6815                    })
6816                    .to_string(),
6817                }))],
6818                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6819                    text: "final".to_string(),
6820                    phase: None,
6821                }))],
6822            };
6823            Ok(Box::pin(stream::iter(events.into_iter().chain(
6824                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6825                    stop_reason: Some("stop".to_string()),
6826                    provider_response_id: None,
6827                }))),
6828            ))))
6829        }
6830    }
6831
6832    struct VerificationGateEngine {
6833        calls: StdMutex<u32>,
6834    }
6835
6836    #[async_trait::async_trait]
6837    impl InferenceEngine for VerificationGateEngine {
6838        fn id(&self) -> String {
6839            roder_api::catalog::PROVIDER_MOCK.to_string()
6840        }
6841
6842        fn capabilities(&self) -> InferenceCapabilities {
6843            InferenceCapabilities::coding_agent_default()
6844        }
6845
6846        async fn list_models(
6847            &self,
6848            _ctx: InferenceProviderContext<'_>,
6849        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6850            Ok(roder_api::catalog::models_for_provider(
6851                roder_api::catalog::PROVIDER_MOCK,
6852                true,
6853            ))
6854        }
6855
6856        async fn stream_turn(
6857            &self,
6858            _ctx: InferenceTurnContext<'_>,
6859            request: AgentInferenceRequest,
6860        ) -> anyhow::Result<InferenceEventStream> {
6861            let mut calls = self.calls.lock().unwrap();
6862            *calls += 1;
6863            let events = match *calls {
6864                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6865                    id: "write-1".to_string(),
6866                    name: "write_file".to_string(),
6867                    arguments: serde_json::json!({
6868                        "path": "src/lib.rs",
6869                        "content": "pub fn answer() -> u8 { 42 }\n"
6870                    })
6871                    .to_string(),
6872                }))],
6873                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6874                    text: "done too early".to_string(),
6875                    phase: None,
6876                }))],
6877                3 if request.transcript.iter().any(|item| {
6878                    matches!(
6879                        item,
6880                        TranscriptItem::UserMessage(message)
6881                            if message.text.contains("Verification gate blocked final completion")
6882                    )
6883                }) =>
6884                {
6885                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6886                        id: "verify-1".to_string(),
6887                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6888                        arguments: serde_json::json!({
6889                            "originalTask": "write code",
6890                            "changedFiles": ["src/lib.rs"],
6891                            "toolEvidence": ["write_file wrote src/lib.rs"],
6892                            "testsRun": ["cargo test -p roder-core verification_gate"],
6893                            "openGaps": [],
6894                            "status": "completed"
6895                        })
6896                        .to_string(),
6897                    }))]
6898                }
6899                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6900                    text: "verified final".to_string(),
6901                    phase: None,
6902                }))],
6903            };
6904            Ok(Box::pin(stream::iter(events.into_iter().chain(
6905                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6906                    stop_reason: Some("stop".to_string()),
6907                    provider_response_id: None,
6908                }))),
6909            ))))
6910        }
6911    }
6912
6913    struct SpeedPolicyEngine {
6914        calls: StdMutex<u32>,
6915        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6916    }
6917
6918    #[async_trait::async_trait]
6919    impl InferenceEngine for SpeedPolicyEngine {
6920        fn id(&self) -> String {
6921            roder_api::catalog::PROVIDER_MOCK.to_string()
6922        }
6923
6924        fn capabilities(&self) -> InferenceCapabilities {
6925            InferenceCapabilities::coding_agent_default()
6926        }
6927
6928        async fn list_models(
6929            &self,
6930            _ctx: InferenceProviderContext<'_>,
6931        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6932            Ok(roder_api::catalog::models_for_provider(
6933                roder_api::catalog::PROVIDER_MOCK,
6934                true,
6935            ))
6936        }
6937
6938        async fn stream_turn(
6939            &self,
6940            _ctx: InferenceTurnContext<'_>,
6941            request: AgentInferenceRequest,
6942        ) -> anyhow::Result<InferenceEventStream> {
6943            self.requests.lock().unwrap().push(request.clone());
6944            let mut calls = self.calls.lock().unwrap();
6945            *calls += 1;
6946            let events = match *calls {
6947                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6948                    id: "write-1".to_string(),
6949                    name: "write_file".to_string(),
6950                    arguments: serde_json::json!({
6951                        "path": "src/lib.rs",
6952                        "content": "pub fn answer() -> u8 { 42 }\n"
6953                    })
6954                    .to_string(),
6955                }))],
6956                3 if request.transcript.iter().any(|item| {
6957                    matches!(
6958                        item,
6959                        TranscriptItem::UserMessage(message)
6960                            if message.text.contains("Verification gate blocked final completion")
6961                    )
6962                }) =>
6963                {
6964                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6965                        id: "verify-1".to_string(),
6966                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6967                        arguments: serde_json::json!({
6968                            "originalTask": "write code",
6969                            "changedFiles": ["src/lib.rs"],
6970                            "toolEvidence": ["write_file wrote src/lib.rs"],
6971                            "testsRun": ["cargo test -p roder-core speed_policy"],
6972                            "openGaps": [],
6973                            "status": "completed"
6974                        })
6975                        .to_string(),
6976                    }))]
6977                }
6978                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6979                    text: "done".to_string(),
6980                    phase: None,
6981                }))],
6982            };
6983            Ok(Box::pin(stream::iter(events.into_iter().chain(
6984                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6985                    stop_reason: Some("stop".to_string()),
6986                    provider_response_id: None,
6987                }))),
6988            ))))
6989        }
6990    }
6991
6992    struct SwitchCaptureEngine {
6993        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6994    }
6995
6996    #[async_trait::async_trait]
6997    impl InferenceEngine for SwitchCaptureEngine {
6998        fn id(&self) -> String {
6999            roder_api::catalog::PROVIDER_MOCK.to_string()
7000        }
7001
7002        fn capabilities(&self) -> InferenceCapabilities {
7003            InferenceCapabilities::coding_agent_default()
7004        }
7005
7006        async fn list_models(
7007            &self,
7008            _ctx: InferenceProviderContext<'_>,
7009        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7010            Ok(roder_api::catalog::models_for_provider(
7011                roder_api::catalog::PROVIDER_MOCK,
7012                true,
7013            ))
7014        }
7015
7016        async fn stream_turn(
7017            &self,
7018            _ctx: InferenceTurnContext<'_>,
7019            request: AgentInferenceRequest,
7020        ) -> anyhow::Result<InferenceEventStream> {
7021            self.requests.lock().unwrap().push(request);
7022            Ok(Box::pin(stream::iter(vec![
7023                Ok(InferenceEvent::MessageDelta(MessageDelta {
7024                    text: "done".to_string(),
7025                    phase: None,
7026                })),
7027                Ok(InferenceEvent::Completed(CompletionMetadata {
7028                    stop_reason: Some("stop".to_string()),
7029                    provider_response_id: None,
7030                })),
7031            ])))
7032        }
7033    }
7034
7035    struct DeadlineEngine;
7036
7037    #[async_trait::async_trait]
7038    impl InferenceEngine for DeadlineEngine {
7039        fn id(&self) -> String {
7040            roder_api::catalog::PROVIDER_MOCK.to_string()
7041        }
7042
7043        fn capabilities(&self) -> InferenceCapabilities {
7044            InferenceCapabilities::coding_agent_default()
7045        }
7046
7047        async fn list_models(
7048            &self,
7049            _ctx: InferenceProviderContext<'_>,
7050        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7051            Ok(Vec::new())
7052        }
7053
7054        async fn stream_turn(
7055            &self,
7056            _ctx: InferenceTurnContext<'_>,
7057            _request: AgentInferenceRequest,
7058        ) -> anyhow::Result<InferenceEventStream> {
7059            Ok(Box::pin(stream::once(async {
7060                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7061                Ok(InferenceEvent::MessageDelta(MessageDelta {
7062                    text: "too late".to_string(),
7063                    phase: None,
7064                }))
7065            })))
7066        }
7067    }
7068
7069    struct WriteFileContributor;
7070
7071    impl ToolContributor for WriteFileContributor {
7072        fn id(&self) -> String {
7073            "test-write".to_string()
7074        }
7075
7076        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7077            registry.register(Arc::new(WriteFileTool))
7078        }
7079    }
7080
7081    struct WriteFileTool;
7082
7083    #[async_trait::async_trait]
7084    impl ToolExecutor for WriteFileTool {
7085        fn spec(&self) -> ToolSpec {
7086            ToolSpec {
7087                name: "write_file".to_string(),
7088                description: "Write a test file.".to_string(),
7089                parameters: serde_json::json!({
7090                    "type": "object",
7091                    "properties": {
7092                        "path": { "type": "string" },
7093                        "content": { "type": "string" }
7094                    },
7095                    "required": ["path", "content"],
7096                    "additionalProperties": false
7097                }),
7098            }
7099        }
7100
7101        async fn execute(
7102            &self,
7103            _ctx: ToolExecutionContext,
7104            call: ToolCall,
7105        ) -> anyhow::Result<ToolResult> {
7106            let path = call
7107                .arguments
7108                .get("path")
7109                .and_then(serde_json::Value::as_str)
7110                .unwrap_or("src/lib.rs");
7111            Ok(ToolResult {
7112                id: call.id,
7113                name: call.name,
7114                text: format!("wrote {path}"),
7115                data: serde_json::json!({ "path": path }),
7116                is_error: false,
7117            })
7118        }
7119    }
7120
7121    struct ProfileToolContributor;
7122
7123    impl ToolContributor for ProfileToolContributor {
7124        fn id(&self) -> String {
7125            "profile-tools".to_string()
7126        }
7127
7128        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7129            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7130                registry.register(Arc::new(ProfileTool {
7131                    name: name.to_string(),
7132                }))?;
7133            }
7134            Ok(())
7135        }
7136    }
7137
7138    struct ProfileTool {
7139        name: String,
7140    }
7141
7142    #[async_trait::async_trait]
7143    impl ToolExecutor for ProfileTool {
7144        fn spec(&self) -> ToolSpec {
7145            ToolSpec {
7146                name: self.name.clone(),
7147                description: format!("{} test tool", self.name),
7148                parameters: serde_json::json!({
7149                    "type": "object",
7150                    "properties": {
7151                        "path": { "type": "string" },
7152                        "content": { "type": "string" }
7153                    },
7154                    "required": ["path", "content"],
7155                    "additionalProperties": false
7156                }),
7157            }
7158        }
7159
7160        async fn execute(
7161            &self,
7162            _ctx: ToolExecutionContext,
7163            call: ToolCall,
7164        ) -> anyhow::Result<ToolResult> {
7165            Ok(ToolResult {
7166                id: call.id,
7167                name: call.name,
7168                text: "ok".to_string(),
7169                data: serde_json::json!({}),
7170                is_error: false,
7171            })
7172        }
7173    }
7174
7175    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7176        ModelHarnessProfile {
7177            model: model.to_string(),
7178            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7179            provider_family: ProviderFamily::OpenAi,
7180            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7181            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7182            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7183            reasoning: ModelProfileReasoning {
7184                orientation: Some(REASONING_LOW.to_string()),
7185                execution: Some(REASONING_LOW.to_string()),
7186                verification: Some(REASONING_LOW.to_string()),
7187                recovery: Some(REASONING_LOW.to_string()),
7188            },
7189            parallel_tool_calls: Some(false),
7190            auto_compact_token_limit: Some(123_000),
7191        }
7192    }
7193
7194    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7195        let captured = Arc::new(StdMutex::new(None));
7196        let mut builder = ExtensionRegistryBuilder::new();
7197        builder.inference_engine(Arc::new(CapturingEngine {
7198            request: captured.clone(),
7199        }));
7200        builder.tool_contributor(Arc::new(ProfileToolContributor));
7201        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7202        let mut rx = runtime.subscribe_events();
7203        let turn_id = runtime
7204            .start_turn(StartTurnRequest {
7205                thread_id: "thread-model-profile".to_string(),
7206                message: "use profile knobs".to_string(),
7207                images: Vec::new(),
7208                provider_override: None,
7209                model_override: None,
7210                reasoning_override: None,
7211                workspace: test_workspace(),
7212                instructions: InstructionBundle {
7213                    system: None,
7214                    developer: Some("base developer".to_string()),
7215                    developer_context: None,
7216                },
7217                developer_context: None,
7218                task_ledger_required: false,
7219            })
7220            .await
7221            .unwrap();
7222
7223        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7224            loop {
7225                let envelope = rx.recv().await.unwrap();
7226                if envelope.turn_id.as_deref() != Some(&turn_id) {
7227                    continue;
7228                }
7229                match envelope.event {
7230                    RoderEvent::TurnCompleted(_) => break,
7231                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7232                    _ => {}
7233                }
7234            }
7235        })
7236        .await
7237        .unwrap();
7238
7239        captured.lock().unwrap().clone().unwrap()
7240    }
7241
7242    struct ToolThenStopEngine {
7243        calls: StdMutex<u32>,
7244    }
7245
7246    #[async_trait::async_trait]
7247    impl InferenceEngine for ToolThenStopEngine {
7248        fn id(&self) -> String {
7249            roder_api::catalog::PROVIDER_MOCK.to_string()
7250        }
7251
7252        fn capabilities(&self) -> InferenceCapabilities {
7253            InferenceCapabilities::coding_agent_default()
7254        }
7255
7256        async fn list_models(
7257            &self,
7258            _ctx: InferenceProviderContext<'_>,
7259        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7260            Ok(roder_api::catalog::models_for_provider(
7261                roder_api::catalog::PROVIDER_MOCK,
7262                true,
7263            ))
7264        }
7265
7266        async fn stream_turn(
7267            &self,
7268            _ctx: InferenceTurnContext<'_>,
7269            _request: AgentInferenceRequest,
7270        ) -> anyhow::Result<InferenceEventStream> {
7271            let mut calls = self.calls.lock().unwrap();
7272            *calls += 1;
7273            let events = match *calls {
7274                1 => vec![
7275                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7276                        id: "write-1".to_string(),
7277                        name: "write_file".to_string(),
7278                        arguments: serde_json::json!({
7279                            "path": "src/lib.rs",
7280                            "content": "pub fn answer() -> u8 { 42 }\n"
7281                        })
7282                        .to_string(),
7283                    })),
7284                    Ok(InferenceEvent::Completed(CompletionMetadata {
7285                        stop_reason: Some("tool_use".to_string()),
7286                        provider_response_id: None,
7287                    })),
7288                ],
7289                _ => vec![
7290                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7291                        text: "final".to_string(),
7292                        phase: None,
7293                    })),
7294                    Ok(InferenceEvent::Completed(CompletionMetadata {
7295                        stop_reason: Some("end_turn".to_string()),
7296                        provider_response_id: None,
7297                    })),
7298                ],
7299            };
7300            Ok(Box::pin(stream::iter(events)))
7301        }
7302    }
7303
7304    #[tokio::test]
7305    async fn turn_completed_reports_terminal_step_finish_reason() {
7306        let mut builder = ExtensionRegistryBuilder::new();
7307        builder.inference_engine(Arc::new(ToolThenStopEngine {
7308            calls: StdMutex::new(0),
7309        }));
7310        builder.tool_contributor(Arc::new(WriteFileContributor));
7311        let runtime = Arc::new(
7312            Runtime::new(
7313                builder.build().unwrap(),
7314                RuntimeConfig {
7315                    policy_mode: PolicyMode::Bypass,
7316                    agent_swarm_mode: false,
7317                    ..RuntimeConfig::default()
7318                },
7319            )
7320            .unwrap(),
7321        );
7322        let mut rx = runtime.subscribe_events();
7323        let turn_id = runtime
7324            .start_turn(StartTurnRequest {
7325                thread_id: "thread-finish-reason".to_string(),
7326                message: "write then finish".to_string(),
7327                images: Vec::new(),
7328                provider_override: None,
7329                model_override: None,
7330                reasoning_override: None,
7331                workspace: test_workspace(),
7332                instructions: InstructionBundle {
7333                    system: None,
7334                    developer: None,
7335                    developer_context: None,
7336                },
7337                developer_context: None,
7338                task_ledger_required: false,
7339            })
7340            .await
7341            .unwrap();
7342
7343        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7344            loop {
7345                let envelope = rx.recv().await.unwrap();
7346                if envelope.turn_id.as_deref() != Some(&turn_id) {
7347                    continue;
7348                }
7349                match envelope.event {
7350                    RoderEvent::TurnCompleted(event) => break event,
7351                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7352                    _ => {}
7353                }
7354            }
7355        })
7356        .await
7357        .unwrap();
7358
7359        // The mid-turn tool_use stop reason must not leak; the terminal
7360        // end_turn step decides the turn's finish reason.
7361        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7362    }
7363
7364    #[tokio::test]
7365    async fn inference_router_selection_changes_request_model_and_records_event() {
7366        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7367        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7368        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7369        let selected = ModelSelection {
7370            provider: "routed-provider".to_string(),
7371            model: "routed-model".to_string(),
7372        };
7373        let default = ModelSelection {
7374            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7375            model: "mock".to_string(),
7376        };
7377        let decision = InferenceRoutingDecision {
7378            reasoning: Some(ReasoningConfig {
7379                enabled: true,
7380                level: Some(REASONING_LOW.to_string()),
7381            }),
7382            confidence: Some(0.91),
7383            baseline: Some(default.clone()),
7384            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7385                "intent", "routine",
7386            )],
7387            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7388        };
7389
7390        let mut builder = ExtensionRegistryBuilder::new();
7391        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7392            id: roder_api::catalog::PROVIDER_MOCK,
7393            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7394            requests: default_requests.clone(),
7395        }));
7396        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7397            id: "routed-provider",
7398            models: vec![routing_test_model(
7399                "routed-model",
7400                &[REASONING_LOW, REASONING_MEDIUM],
7401            )],
7402            requests: routed_requests.clone(),
7403        }));
7404        builder.inference_router(Arc::new(StaticRouter {
7405            id: "test-router",
7406            decision,
7407            contexts: contexts.clone(),
7408        }));
7409        let thread_root =
7410            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7411        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7412            base_path: thread_root.clone(),
7413        }));
7414        let runtime = Arc::new(
7415            Runtime::new(
7416                builder.build().unwrap(),
7417                RuntimeConfig {
7418                    default_provider: default.provider.clone(),
7419                    default_model: default.model.clone(),
7420                    ..RuntimeConfig::default()
7421                },
7422            )
7423            .unwrap(),
7424        );
7425        let thread_id = runtime
7426            .create_thread_with(CreateThreadRequest {
7427                title: Some("Routing auto".to_string()),
7428                workspace: test_workspace(),
7429                workspace_id: None,
7430                root_id: None,
7431                provider: Some(default.provider.clone()),
7432                model: Some(default.model.clone()),
7433                tool_allowlist: Vec::new(),
7434                developer_instructions: None,
7435                external_tools: Vec::new(),
7436                selection_mode: Some(ModelSelectionMode::auto(
7437                    "test-router:coding",
7438                    "test-router",
7439                    "Auto: Coding",
7440                    default.clone(),
7441                    Some("coding".to_string()),
7442                    None,
7443                )),
7444                runner: None,
7445            })
7446            .await
7447            .unwrap()
7448            .thread_id;
7449        let mut rx = runtime.subscribe_events();
7450        let turn_id = runtime
7451            .start_turn(StartTurnRequest {
7452                thread_id: thread_id.clone(),
7453                message: "small cleanup".to_string(),
7454                images: Vec::new(),
7455                provider_override: None,
7456                model_override: None,
7457                reasoning_override: None,
7458                workspace: test_workspace(),
7459                instructions: InstructionBundle::default(),
7460                developer_context: None,
7461                task_ledger_required: false,
7462            })
7463            .await
7464            .unwrap();
7465
7466        let mut routing_event = None;
7467        let mut inference_started = None;
7468        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7469            loop {
7470                let envelope = rx.recv().await.unwrap();
7471                if envelope.turn_id.as_deref() != Some(&turn_id) {
7472                    continue;
7473                }
7474                match envelope.event {
7475                    RoderEvent::InferenceRoutingDecision(event) => {
7476                        routing_event = Some(event);
7477                    }
7478                    RoderEvent::InferenceStarted(event) => {
7479                        inference_started = Some(event);
7480                    }
7481                    RoderEvent::TurnCompleted(_) => break,
7482                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7483                    _ => {}
7484                }
7485            }
7486        })
7487        .await
7488        .unwrap();
7489
7490        assert!(default_requests.lock().unwrap().is_empty());
7491        let routed_requests = routed_requests.lock().unwrap();
7492        assert_eq!(routed_requests.len(), 1);
7493        assert_eq!(routed_requests[0].model, selected);
7494        assert_eq!(
7495            routed_requests[0].reasoning.level.as_deref(),
7496            Some(REASONING_LOW)
7497        );
7498        assert_eq!(
7499            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7500            "selected"
7501        );
7502
7503        let routing_event = routing_event.expect("routing decision event");
7504        assert_eq!(routing_event.default_selection, default);
7505        assert_eq!(routing_event.selected_selection, selected);
7506        assert_eq!(
7507            routing_event.decision.outcome,
7508            InferenceRoutingOutcome::Selected
7509        );
7510        assert_eq!(
7511            inference_started.expect("inference started event").model,
7512            selected
7513        );
7514
7515        let contexts = contexts.lock().unwrap();
7516        assert_eq!(contexts.len(), 1);
7517        assert_eq!(contexts[0].default_selection, default);
7518        assert_eq!(contexts[0].candidates.len(), 2);
7519        assert!(
7520            contexts[0]
7521                .signals
7522                .iter()
7523                .any(|signal| signal.key == "profile" && signal.value == "coding")
7524        );
7525        let _ = std::fs::remove_dir_all(thread_root);
7526    }
7527
7528    #[tokio::test]
7529    async fn inference_router_is_bypassed_for_explicit_selection() {
7530        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7531        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7532        let selected = ModelSelection {
7533            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7534            model: "mock".to_string(),
7535        };
7536
7537        let mut builder = ExtensionRegistryBuilder::new();
7538        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7539            id: roder_api::catalog::PROVIDER_MOCK,
7540            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7541            requests: requests.clone(),
7542        }));
7543        builder.inference_router(Arc::new(StaticRouter {
7544            id: "test-router",
7545            decision: InferenceRoutingDecision::selected(
7546                "test-router",
7547                ModelSelection {
7548                    provider: "missing".to_string(),
7549                    model: "missing".to_string(),
7550                },
7551                "would route if called",
7552            ),
7553            contexts: contexts.clone(),
7554        }));
7555        let thread_root =
7556            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7557        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7558            base_path: thread_root.clone(),
7559        }));
7560        let runtime = Arc::new(
7561            Runtime::new(
7562                builder.build().unwrap(),
7563                RuntimeConfig {
7564                    default_provider: selected.provider.clone(),
7565                    default_model: selected.model.clone(),
7566                    inference_router: RuntimeInferenceRouterConfig {
7567                        enabled: true,
7568                        router_id: Some("test-router".to_string()),
7569                    },
7570                    ..RuntimeConfig::default()
7571                },
7572            )
7573            .unwrap(),
7574        );
7575        let thread_id = runtime
7576            .create_thread_with(CreateThreadRequest {
7577                title: Some("Routing explicit".to_string()),
7578                workspace: test_workspace(),
7579                workspace_id: None,
7580                root_id: None,
7581                provider: Some(selected.provider.clone()),
7582                model: Some(selected.model.clone()),
7583                tool_allowlist: Vec::new(),
7584                developer_instructions: None,
7585                external_tools: Vec::new(),
7586                selection_mode: Some(ModelSelectionMode::auto(
7587                    "test-router:default",
7588                    "test-router",
7589                    "Auto",
7590                    selected.clone(),
7591                    None,
7592                    None,
7593                )),
7594                runner: None,
7595            })
7596            .await
7597            .unwrap()
7598            .thread_id;
7599        let mut rx = runtime.subscribe_events();
7600        let turn_id = runtime
7601            .start_turn(StartTurnRequest {
7602                thread_id,
7603                message: "use explicit selection".to_string(),
7604                images: Vec::new(),
7605                provider_override: Some(selected.provider.clone()),
7606                model_override: Some(selected.model.clone()),
7607                reasoning_override: None,
7608                workspace: test_workspace(),
7609                instructions: InstructionBundle::default(),
7610                developer_context: None,
7611                task_ledger_required: false,
7612            })
7613            .await
7614            .unwrap();
7615
7616        let mut saw_routing_event = false;
7617        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7618            loop {
7619                let envelope = rx.recv().await.unwrap();
7620                if envelope.turn_id.as_deref() != Some(&turn_id) {
7621                    continue;
7622                }
7623                match envelope.event {
7624                    RoderEvent::InferenceRoutingDecision(_) => {
7625                        saw_routing_event = true;
7626                    }
7627                    RoderEvent::TurnCompleted(_) => break,
7628                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7629                    _ => {}
7630                }
7631            }
7632        })
7633        .await
7634        .unwrap();
7635
7636        assert!(!saw_routing_event);
7637        assert!(contexts.lock().unwrap().is_empty());
7638        let requests = requests.lock().unwrap();
7639        assert_eq!(requests.len(), 1);
7640        assert_eq!(requests[0].model, selected);
7641        let _ = std::fs::remove_dir_all(thread_root);
7642    }
7643
7644    #[tokio::test]
7645    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7646        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7647        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7648        let selected = ModelSelection {
7649            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7650            model: "mock".to_string(),
7651        };
7652
7653        let mut builder = ExtensionRegistryBuilder::new();
7654        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7655            id: roder_api::catalog::PROVIDER_MOCK,
7656            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7657            requests: requests.clone(),
7658        }));
7659        builder.inference_router(Arc::new(StaticRouter {
7660            id: "test-router",
7661            decision: InferenceRoutingDecision::selected(
7662                "test-router",
7663                ModelSelection {
7664                    provider: "missing".to_string(),
7665                    model: "missing".to_string(),
7666                },
7667                "would route if called",
7668            ),
7669            contexts: contexts.clone(),
7670        }));
7671        let thread_root =
7672            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7673        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7674            base_path: thread_root.clone(),
7675        }));
7676        let runtime = Arc::new(
7677            Runtime::new(
7678                builder.build().unwrap(),
7679                RuntimeConfig {
7680                    default_provider: selected.provider.clone(),
7681                    default_model: selected.model.clone(),
7682                    inference_router: RuntimeInferenceRouterConfig {
7683                        enabled: true,
7684                        router_id: Some("test-router".to_string()),
7685                    },
7686                    ..RuntimeConfig::default()
7687                },
7688            )
7689            .unwrap(),
7690        );
7691        let thread_id = runtime
7692            .create_thread_with(CreateThreadRequest {
7693                title: Some("Routing manual".to_string()),
7694                workspace: test_workspace(),
7695                workspace_id: None,
7696                root_id: None,
7697                provider: Some(selected.provider.clone()),
7698                model: Some(selected.model.clone()),
7699                tool_allowlist: Vec::new(),
7700                developer_instructions: None,
7701                external_tools: Vec::new(),
7702                selection_mode: Some(ModelSelectionMode::manual(
7703                    selected.provider.clone(),
7704                    selected.model.clone(),
7705                    None,
7706                )),
7707                runner: None,
7708            })
7709            .await
7710            .unwrap()
7711            .thread_id;
7712        let mut rx = runtime.subscribe_events();
7713        let turn_id = runtime
7714            .start_turn(StartTurnRequest {
7715                thread_id,
7716                message: "use selected manual model".to_string(),
7717                images: Vec::new(),
7718                provider_override: None,
7719                model_override: None,
7720                reasoning_override: None,
7721                workspace: test_workspace(),
7722                instructions: InstructionBundle::default(),
7723                developer_context: None,
7724                task_ledger_required: false,
7725            })
7726            .await
7727            .unwrap();
7728
7729        let mut saw_routing_event = false;
7730        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7731            loop {
7732                let envelope = rx.recv().await.unwrap();
7733                if envelope.turn_id.as_deref() != Some(&turn_id) {
7734                    continue;
7735                }
7736                match envelope.event {
7737                    RoderEvent::InferenceRoutingDecision(_) => {
7738                        saw_routing_event = true;
7739                    }
7740                    RoderEvent::TurnCompleted(_) => break,
7741                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7742                    _ => {}
7743                }
7744            }
7745        })
7746        .await
7747        .unwrap();
7748
7749        assert!(!saw_routing_event);
7750        assert!(contexts.lock().unwrap().is_empty());
7751        let requests = requests.lock().unwrap();
7752        assert_eq!(requests.len(), 1);
7753        assert_eq!(requests[0].model, selected);
7754        let _ = std::fs::remove_dir_all(thread_root);
7755    }
7756
7757    #[test]
7758    fn enabled_inference_router_requires_registered_router() {
7759        let mut builder = ExtensionRegistryBuilder::new();
7760        builder.inference_engine(Arc::new(FakeInferenceEngine));
7761
7762        let err = match Runtime::new(
7763            builder.build().unwrap(),
7764            RuntimeConfig {
7765                inference_router: RuntimeInferenceRouterConfig {
7766                    enabled: true,
7767                    router_id: Some("missing-router".to_string()),
7768                },
7769                ..RuntimeConfig::default()
7770            },
7771        ) {
7772            Ok(_) => panic!("runtime should reject unknown inference router"),
7773            Err(err) => err,
7774        };
7775
7776        assert!(
7777            err.to_string()
7778                .contains("inference router \"missing-router\" is not registered")
7779        );
7780    }
7781
7782    #[tokio::test]
7783    async fn model_profile_routes_request_knobs_to_next_inference() {
7784        let request = captured_profile_request(RuntimeConfig {
7785            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7786            default_model: "gpt-5.5".to_string(),
7787            model_profiles: std::collections::HashMap::from([(
7788                "gpt-5.5".to_string(),
7789                test_model_profile("gpt-5.5"),
7790            )]),
7791            ..RuntimeConfig::default()
7792        })
7793        .await;
7794
7795        let tool_names = request
7796            .tools
7797            .iter()
7798            .map(|tool| tool.name.as_str())
7799            .collect::<Vec<_>>();
7800        assert!(tool_names.contains(&"apply_patch"));
7801        assert!(tool_names.contains(&"edit"));
7802        assert!(tool_names.contains(&"multi_edit"));
7803        assert!(tool_names.contains(&"write_file"));
7804        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7805        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7806        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7807        assert!(
7808            request
7809                .instructions
7810                .developer
7811                .as_deref()
7812                .unwrap_or_default()
7813                .contains("Use the provided context as the current working set")
7814        );
7815        assert_eq!(
7816            request
7817                .metadata
7818                .pointer("/modelProfile/schemaPolicy")
7819                .and_then(serde_json::Value::as_str),
7820            Some("standard_required_first")
7821        );
7822    }
7823
7824    #[tokio::test]
7825    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7826        let request = captured_profile_request(RuntimeConfig {
7827            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7828            default_model: "gpt-5.6-sol".to_string(),
7829            reasoning: Some(REASONING_ULTRA.to_string()),
7830            ..RuntimeConfig::default()
7831        })
7832        .await;
7833
7834        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
7835        let developer = request
7836            .instructions
7837            .developer
7838            .as_deref()
7839            .expect("ultra developer instructions");
7840        assert!(developer.contains("Proactive multi-agent delegation is active"));
7841    }
7842
7843    #[tokio::test]
7844    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
7845        let request = captured_profile_request(RuntimeConfig {
7846            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7847            default_model: "gpt-5.6-sol".to_string(),
7848            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
7849            ..RuntimeConfig::default()
7850        })
7851        .await;
7852
7853        let developer = request
7854            .instructions
7855            .developer
7856            .as_deref()
7857            .expect("sol developer instructions");
7858        assert!(developer.contains("Do not spawn sub-agents unless"));
7859        assert!(!developer.contains("Proactive multi-agent delegation is active"));
7860    }
7861
7862    #[tokio::test]
7863    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
7864        let request = captured_profile_request(RuntimeConfig {
7865            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7866            default_model: "gpt-5.6-luna".to_string(),
7867            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
7868            ..RuntimeConfig::default()
7869        })
7870        .await;
7871
7872        let developer = request
7873            .instructions
7874            .developer
7875            .as_deref()
7876            .unwrap_or_default();
7877        assert!(!developer.contains("Do not spawn sub-agents unless"));
7878        assert!(!developer.contains("Proactive multi-agent delegation is active"));
7879    }
7880
7881    #[tokio::test]
7882    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
7883        let captured = Arc::new(StdMutex::new(None));
7884        let mut builder = ExtensionRegistryBuilder::new();
7885        builder.inference_engine(Arc::new(CapturingEngine {
7886            request: captured.clone(),
7887        }));
7888        let runtime = Arc::new(
7889            Runtime::new(
7890                builder.build().unwrap(),
7891                RuntimeConfig {
7892                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7893                    default_model: "gpt-5.5".to_string(),
7894                    ..RuntimeConfig::default()
7895                },
7896            )
7897            .unwrap(),
7898        );
7899
7900        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
7901            let mut rx = runtime.subscribe_events();
7902            let turn_id = runtime
7903                .start_turn(StartTurnRequest {
7904                    thread_id: "thread-turn-context".to_string(),
7905                    message: "hello".to_string(),
7906                    images: Vec::new(),
7907                    provider_override: None,
7908                    model_override: None,
7909                    reasoning_override: None,
7910                    workspace: test_workspace(),
7911                    instructions: InstructionBundle::default(),
7912                    developer_context,
7913                    task_ledger_required: false,
7914                })
7915                .await
7916                .unwrap();
7917            tokio::time::timeout(std::time::Duration::from_secs(5), async {
7918                loop {
7919                    let envelope = rx.recv().await.unwrap();
7920                    if envelope.turn_id.as_deref() != Some(&turn_id) {
7921                        continue;
7922                    }
7923                    match envelope.event {
7924                        RoderEvent::TurnCompleted(_) => break,
7925                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7926                        _ => {}
7927                    }
7928                }
7929            })
7930            .await
7931            .unwrap();
7932        }
7933
7934        run_turn(
7935            &runtime,
7936            Some("Connected accounts: example-service.".to_string()),
7937        )
7938        .await;
7939        let request = captured.lock().unwrap().clone().unwrap();
7940        assert_eq!(
7941            request.instructions.developer_context.as_deref(),
7942            Some("Connected accounts: example-service.")
7943        );
7944
7945        // The context is per-turn only: the next turn on the same thread
7946        // without a developerContext must not see the previous one.
7947        run_turn(&runtime, None).await;
7948        let request = captured.lock().unwrap().clone().unwrap();
7949        assert_eq!(request.instructions.developer_context, None);
7950    }
7951
7952    #[tokio::test]
7953    async fn tool_search_overrides_route_to_next_inference_request() {
7954        let request = captured_profile_request(RuntimeConfig {
7955            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7956            default_model: "gpt-5.4".to_string(),
7957            tool_search: ToolSearchConfig {
7958                mode: roder_api::inference::ToolSearchMode::Auto,
7959                max_catalog_items: Some(100),
7960                ..ToolSearchConfig::default()
7961            },
7962            provider_tool_search: std::collections::HashMap::from([(
7963                roder_api::catalog::PROVIDER_MOCK.to_string(),
7964                roder_api::inference::ToolSearchConfigOverlay {
7965                    include_skills: Some(false),
7966                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
7967                    ..Default::default()
7968                },
7969            )]),
7970            model_tool_search: std::collections::HashMap::from([(
7971                "gpt-5.4".to_string(),
7972                roder_api::inference::ToolSearchConfigOverlay {
7973                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
7974                    max_catalog_items: Some(25),
7975                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
7976                    ..Default::default()
7977                },
7978            )]),
7979            ..RuntimeConfig::default()
7980        })
7981        .await;
7982
7983        assert_eq!(
7984            request.runtime.tool_search.mode,
7985            roder_api::inference::ToolSearchMode::ProviderNative
7986        );
7987        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
7988        assert_eq!(
7989            request.runtime.tool_search.provider_variant,
7990            roder_api::inference::ToolSearchProviderVariant::Bm25
7991        );
7992    }
7993
7994    #[tokio::test]
7995    async fn context_entrypoint_hints_use_turn_workspace() {
7996        let process_workspace = runtime_test_workspace("entrypoint-process");
7997        let thread_workspace = runtime_test_workspace("entrypoint-thread");
7998        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
7999        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8000        std::fs::write(
8001            process_workspace.join("src/sidebar-thread-groups.ts"),
8002            "export const desktopLeak = true;\n",
8003        )
8004        .unwrap();
8005        std::fs::write(
8006            thread_workspace.join("src/voice-plan-feedback.ts"),
8007            "export const voicePlanFeedback = true;\n",
8008        )
8009        .unwrap();
8010
8011        let captured = Arc::new(StdMutex::new(None));
8012        let mut builder = ExtensionRegistryBuilder::new();
8013        builder.inference_engine(Arc::new(CapturingEngine {
8014            request: captured.clone(),
8015        }));
8016        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8017            process_workspace.clone(),
8018        )));
8019        let runtime = Arc::new(
8020            Runtime::new(
8021                builder.build().unwrap(),
8022                RuntimeConfig {
8023                    workspace: Some(process_workspace.display().to_string()),
8024                    ..RuntimeConfig::default()
8025                },
8026            )
8027            .unwrap(),
8028        );
8029        let mut rx = runtime.subscribe_events();
8030
8031        let turn_id = runtime
8032            .start_turn(StartTurnRequest {
8033                thread_id: "thread-workspace-entrypoint".to_string(),
8034                message: "investigate voice plan feedback".to_string(),
8035                images: Vec::new(),
8036                provider_override: None,
8037                model_override: None,
8038                reasoning_override: None,
8039                workspace: thread_workspace.display().to_string(),
8040                instructions: crate::instructions::default_instructions(),
8041                developer_context: None,
8042                task_ledger_required: false,
8043            })
8044            .await
8045            .unwrap();
8046
8047        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8048            loop {
8049                let envelope = rx.recv().await.unwrap();
8050                if envelope.turn_id.as_deref() != Some(&turn_id) {
8051                    continue;
8052                }
8053                match envelope.event {
8054                    RoderEvent::TurnCompleted(_) => break,
8055                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8056                    _ => {}
8057                }
8058            }
8059        })
8060        .await
8061        .unwrap();
8062
8063        let request = captured.lock().unwrap().clone().expect("captured request");
8064        let transcript_text = request
8065            .transcript
8066            .iter()
8067            .map(|item| match item {
8068                TranscriptItem::UserMessage(message) => message.text.as_str(),
8069                _ => "",
8070            })
8071            .collect::<Vec<_>>()
8072            .join("\n");
8073        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8074        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8075
8076        let _ = std::fs::remove_dir_all(process_workspace);
8077        let _ = std::fs::remove_dir_all(thread_workspace);
8078    }
8079
8080    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8081        let path =
8082            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8083        let _ = std::fs::remove_dir_all(&path);
8084        std::fs::create_dir_all(&path).unwrap();
8085        path
8086    }
8087
8088    #[tokio::test]
8089    async fn model_profile_user_model_knobs_override_profile_defaults() {
8090        let request = captured_profile_request(RuntimeConfig {
8091            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8092            default_model: "gpt-5.5".to_string(),
8093            reasoning: Some(REASONING_HIGH.to_string()),
8094            auto_compact_token_limit: Some(999),
8095            model_edit_tools: std::collections::HashMap::from([(
8096                "gpt-5.5".to_string(),
8097                EDIT_TOOL_PATCH.to_string(),
8098            )]),
8099            model_parallel_tool_calls: std::collections::HashMap::from([(
8100                "gpt-5.5".to_string(),
8101                true,
8102            )]),
8103            model_profiles: std::collections::HashMap::from([(
8104                "gpt-5.5".to_string(),
8105                test_model_profile("gpt-5.5"),
8106            )]),
8107            ..RuntimeConfig::default()
8108        })
8109        .await;
8110
8111        let tool_names = request
8112            .tools
8113            .iter()
8114            .map(|tool| tool.name.as_str())
8115            .collect::<Vec<_>>();
8116        assert!(tool_names.contains(&"apply_patch"));
8117        assert!(!tool_names.contains(&"edit"));
8118        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8119        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8120        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8121    }
8122
8123    #[tokio::test]
8124    async fn runtime_tool_allowlist_filters_advertised_tools() {
8125        let request = captured_profile_request(RuntimeConfig {
8126            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8127            default_model: "gpt-5.5".to_string(),
8128            tool_allowlist: vec!["edit".to_string()],
8129            model_profiles: std::collections::HashMap::from([(
8130                "gpt-5.5".to_string(),
8131                test_model_profile("gpt-5.5"),
8132            )]),
8133            ..RuntimeConfig::default()
8134        })
8135        .await;
8136
8137        let tool_names = request
8138            .tools
8139            .iter()
8140            .map(|tool| tool.name.as_str())
8141            .collect::<Vec<_>>();
8142        assert_eq!(tool_names, vec!["edit"]);
8143    }
8144
8145    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8146    /// overrides and returns the captured inference request.
8147    async fn captured_thread_override_request(
8148        runtime: &Arc<Runtime>,
8149        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8150        tool_allowlist: Vec<String>,
8151        developer_instructions: Option<String>,
8152        external_tools: Vec<ToolSpec>,
8153    ) -> AgentInferenceRequest {
8154        let thread_id = runtime
8155            .create_thread_with(CreateThreadRequest {
8156                title: Some("Thread overrides".to_string()),
8157                workspace: test_workspace(),
8158                workspace_id: None,
8159                root_id: None,
8160                provider: None,
8161                model: None,
8162                selection_mode: None,
8163                tool_allowlist,
8164                developer_instructions,
8165                external_tools,
8166                runner: None,
8167            })
8168            .await
8169            .unwrap()
8170            .thread_id;
8171        let mut rx = runtime.subscribe_events();
8172        let turn_id = runtime
8173            .start_turn(StartTurnRequest {
8174                thread_id,
8175                message: "hello".to_string(),
8176                images: Vec::new(),
8177                provider_override: None,
8178                model_override: None,
8179                reasoning_override: None,
8180                workspace: test_workspace(),
8181                instructions: crate::default_instructions(),
8182                developer_context: None,
8183                task_ledger_required: false,
8184            })
8185            .await
8186            .unwrap();
8187        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8188            loop {
8189                let envelope = rx.recv().await.unwrap();
8190                if envelope.turn_id.as_deref() != Some(&turn_id) {
8191                    continue;
8192                }
8193                match envelope.event {
8194                    RoderEvent::TurnCompleted(_) => break,
8195                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8196                    _ => {}
8197                }
8198            }
8199        })
8200        .await
8201        .unwrap();
8202        requests.lock().unwrap().pop().expect("captured request")
8203    }
8204
8205    #[tokio::test]
8206    async fn thread_tool_allowlist_filters_only_that_thread() {
8207        let requests = Arc::new(StdMutex::new(Vec::new()));
8208        let thread_root =
8209            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8210        let mut builder = ExtensionRegistryBuilder::new();
8211        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8212            requests: requests.clone(),
8213        }));
8214        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8215            base_path: thread_root.clone(),
8216        }));
8217        builder.tool_contributor(Arc::new(ProfileToolContributor));
8218        let runtime = Arc::new(
8219            Runtime::new(
8220                builder.build().unwrap(),
8221                RuntimeConfig {
8222                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8223                    default_model: "gpt-5.5".to_string(),
8224                    model_profiles: std::collections::HashMap::from([(
8225                        "gpt-5.5".to_string(),
8226                        test_model_profile("gpt-5.5"),
8227                    )]),
8228                    ..RuntimeConfig::default()
8229                },
8230            )
8231            .unwrap(),
8232        );
8233
8234        let allowlisted = captured_thread_override_request(
8235            &runtime,
8236            &requests,
8237            vec!["edit".to_string()],
8238            None,
8239            Vec::new(),
8240        )
8241        .await;
8242        let unrestricted =
8243            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8244                .await;
8245
8246        let allowlisted_names = allowlisted
8247            .tools
8248            .iter()
8249            .map(|tool| tool.name.as_str())
8250            .collect::<Vec<_>>();
8251        assert_eq!(allowlisted_names, vec!["edit"]);
8252        let unrestricted_names = unrestricted
8253            .tools
8254            .iter()
8255            .map(|tool| tool.name.as_str())
8256            .collect::<Vec<_>>();
8257        assert!(unrestricted_names.contains(&"edit"));
8258        assert!(unrestricted_names.len() > 1);
8259
8260        let _ = std::fs::remove_dir_all(thread_root);
8261    }
8262
8263    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8264    fn runtime_with_edit_allowlist(
8265        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8266        thread_root: &std::path::Path,
8267    ) -> Arc<Runtime> {
8268        let mut builder = ExtensionRegistryBuilder::new();
8269        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8270            requests: requests.clone(),
8271        }));
8272        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8273            base_path: thread_root.to_path_buf(),
8274        }));
8275        builder.tool_contributor(Arc::new(ProfileToolContributor));
8276        Arc::new(
8277            Runtime::new(
8278                builder.build().unwrap(),
8279                RuntimeConfig {
8280                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8281                    default_model: "gpt-5.5".to_string(),
8282                    tool_allowlist: vec!["edit".to_string()],
8283                    model_profiles: std::collections::HashMap::from([(
8284                        "gpt-5.5".to_string(),
8285                        test_model_profile("gpt-5.5"),
8286                    )]),
8287                    ..RuntimeConfig::default()
8288                },
8289            )
8290            .unwrap(),
8291        )
8292    }
8293
8294    #[tokio::test]
8295    async fn runtime_and_thread_allowlists_intersect() {
8296        let requests = Arc::new(StdMutex::new(Vec::new()));
8297        let thread_root = std::env::temp_dir().join(format!(
8298            "roder-allowlist-intersect-{}",
8299            uuid::Uuid::new_v4()
8300        ));
8301        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8302
8303        let request = captured_thread_override_request(
8304            &runtime,
8305            &requests,
8306            vec!["edit".to_string(), "write_file".to_string()],
8307            None,
8308            Vec::new(),
8309        )
8310        .await;
8311
8312        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8313        let names = request
8314            .tools
8315            .iter()
8316            .map(|tool| tool.name.as_str())
8317            .collect::<Vec<_>>();
8318        assert_eq!(names, vec!["edit"]);
8319
8320        let _ = std::fs::remove_dir_all(thread_root);
8321    }
8322
8323    #[tokio::test]
8324    async fn route_tool_call_denies_tools_outside_allowlists() {
8325        let requests = Arc::new(StdMutex::new(Vec::new()));
8326        let thread_root =
8327            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8328        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8329        let thread_id = runtime
8330            .create_thread_with(CreateThreadRequest {
8331                title: Some("Dispatch allowlist".to_string()),
8332                workspace: test_workspace(),
8333                workspace_id: None,
8334                root_id: None,
8335                provider: None,
8336                model: None,
8337                selection_mode: None,
8338                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8339                developer_instructions: None,
8340                external_tools: Vec::new(),
8341                runner: None,
8342            })
8343            .await
8344            .unwrap()
8345            .thread_id;
8346
8347        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8348        let result = runtime
8349            .route_tool_call(
8350                &thread_id,
8351                &"turn-allowlist-dispatch".to_string(),
8352                roder_api::inference::ToolCallCompleted {
8353                    id: "call-1".to_string(),
8354                    name: "write_file".to_string(),
8355                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8356                },
8357                None,
8358                None,
8359            )
8360            .await
8361            .unwrap();
8362
8363        assert!(result.is_error);
8364        assert!(
8365            result
8366                .result
8367                .contains("not permitted by the tool allowlist"),
8368            "unexpected result: {}",
8369            result.result
8370        );
8371
8372        let _ = std::fs::remove_dir_all(thread_root);
8373    }
8374
8375    #[tokio::test]
8376    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8377        let requests = Arc::new(StdMutex::new(Vec::new()));
8378        let thread_root =
8379            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8380        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8381
8382        // A response that mixes agent_swarm with another tool is denied wholesale:
8383        // both calls get an error result with retry guidance, and no tool runs.
8384        let results = runtime
8385            .route_tool_calls(
8386                &"thread-swarm".to_string(),
8387                &"turn-swarm".to_string(),
8388                vec![
8389                    roder_api::inference::ToolCallCompleted {
8390                        id: "swarm-1".to_string(),
8391                        name: "agent_swarm".to_string(),
8392                        arguments: "{}".to_string(),
8393                    },
8394                    roder_api::inference::ToolCallCompleted {
8395                        id: "read-1".to_string(),
8396                        name: "read_file".to_string(),
8397                        arguments: "{}".to_string(),
8398                    },
8399                ],
8400                true,
8401                None,
8402                None,
8403            )
8404            .await
8405            .unwrap();
8406
8407        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8408        assert_eq!(results[0].id, "swarm-1");
8409        assert_eq!(results[1].id, "read-1");
8410        for result in &results {
8411            assert!(result.is_error);
8412            assert!(
8413                result.result.contains("only tool call"),
8414                "unexpected result: {}",
8415                result.result
8416            );
8417        }
8418
8419        let _ = std::fs::remove_dir_all(thread_root);
8420    }
8421
8422    #[tokio::test]
8423    async fn route_tool_calls_emits_agent_swarm_started_event() {
8424        let requests = Arc::new(StdMutex::new(Vec::new()));
8425        let thread_root =
8426            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8427        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8428        let mut events = runtime.subscribe_events();
8429
8430        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8431        // the event bus before dispatch, with the child count parsed from args.
8432        let _ = runtime
8433            .route_tool_calls(
8434                &"thread-swarm".to_string(),
8435                &"turn-swarm".to_string(),
8436                vec![roder_api::inference::ToolCallCompleted {
8437                    id: "swarm-1".to_string(),
8438                    name: "agent_swarm".to_string(),
8439                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8440                        .to_string(),
8441                }],
8442                true,
8443                None,
8444                None,
8445            )
8446            .await
8447            .unwrap();
8448
8449        let mut started_child_count = None;
8450        for _ in 0..16 {
8451            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8452                .await
8453                .unwrap()
8454                .unwrap();
8455            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8456                started_child_count = Some(event.child_count);
8457                assert_eq!(event.tool_id, "swarm-1");
8458                break;
8459            }
8460        }
8461        assert_eq!(started_child_count, Some(2));
8462
8463        let _ = std::fs::remove_dir_all(thread_root);
8464    }
8465
8466    #[test]
8467    fn agent_swarm_child_count_sums_items_and_resumes() {
8468        assert_eq!(
8469            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8470            3
8471        );
8472        assert_eq!(
8473            agent_swarm_child_count(
8474                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8475            ),
8476            2
8477        );
8478        // Malformed input is lenient.
8479        assert_eq!(agent_swarm_child_count("not json"), 0);
8480    }
8481
8482    #[test]
8483    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8484        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8485        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8486        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8487        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8488        // Not a swarm result.
8489        assert_eq!(parse_swarm_counts("just text"), None);
8490    }
8491
8492    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8493    struct SignalledFailureEngine {
8494        started: tokio::sync::mpsc::UnboundedSender<()>,
8495        proceed: Arc<tokio::sync::Notify>,
8496    }
8497
8498    #[async_trait::async_trait]
8499    impl InferenceEngine for SignalledFailureEngine {
8500        fn id(&self) -> String {
8501            roder_api::catalog::PROVIDER_MOCK.to_string()
8502        }
8503
8504        fn capabilities(&self) -> InferenceCapabilities {
8505            InferenceCapabilities::coding_agent_default()
8506        }
8507
8508        async fn list_models(
8509            &self,
8510            _ctx: InferenceProviderContext<'_>,
8511        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8512            Ok(roder_api::catalog::models_for_provider(
8513                roder_api::catalog::PROVIDER_MOCK,
8514                true,
8515            ))
8516        }
8517
8518        async fn stream_turn(
8519            &self,
8520            _ctx: InferenceTurnContext<'_>,
8521            _request: AgentInferenceRequest,
8522        ) -> anyhow::Result<InferenceEventStream> {
8523            let _ = self.started.send(());
8524            self.proceed.notified().await;
8525            anyhow::bail!("engine failed mid-turn")
8526        }
8527    }
8528
8529    #[tokio::test]
8530    async fn failed_turn_sweeps_pending_external_tool_calls() {
8531        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8532        let proceed = Arc::new(tokio::sync::Notify::new());
8533        let mut builder = ExtensionRegistryBuilder::new();
8534        builder.inference_engine(Arc::new(SignalledFailureEngine {
8535            started: started_tx,
8536            proceed: proceed.clone(),
8537        }));
8538        let runtime =
8539            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8540        let mut rx = runtime.subscribe_events();
8541        let turn_id = runtime
8542            .start_turn(StartTurnRequest {
8543                thread_id: "thread-sweep".to_string(),
8544                message: "go".to_string(),
8545                images: Vec::new(),
8546                provider_override: None,
8547                model_override: None,
8548                reasoning_override: None,
8549                workspace: test_workspace(),
8550                instructions: InstructionBundle {
8551                    system: None,
8552                    developer: None,
8553                    developer_context: None,
8554                },
8555                developer_context: None,
8556                task_ledger_required: false,
8557            })
8558            .await
8559            .unwrap();
8560        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8561            .await
8562            .unwrap()
8563            .unwrap();
8564
8565        let (tx, _pending_rx) = oneshot::channel();
8566        runtime.pending_external_tool_calls.lock().await.insert(
8567            "exttool-sweep-test".to_string(),
8568            PendingExternalToolCall {
8569                thread_id: "thread-sweep".to_string(),
8570                turn_id: turn_id.clone(),
8571                tool_id: "call-1".to_string(),
8572                tool_name: "acme_lookup".to_string(),
8573                tx,
8574            },
8575        );
8576        proceed.notify_one();
8577
8578        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8579            loop {
8580                let envelope = rx.recv().await.unwrap();
8581                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8582                    && event.request_id == "exttool-sweep-test"
8583                {
8584                    break event.outcome;
8585                }
8586            }
8587        })
8588        .await
8589        .expect("turn failure must resolve pending external tool calls");
8590        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8591        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8592    }
8593
8594    #[tokio::test]
8595    async fn thread_developer_instructions_layer_under_harness_prompt() {
8596        let requests = Arc::new(StdMutex::new(Vec::new()));
8597        let thread_root = std::env::temp_dir().join(format!(
8598            "roder-thread-instructions-{}",
8599            uuid::Uuid::new_v4()
8600        ));
8601        let mut builder = ExtensionRegistryBuilder::new();
8602        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8603            requests: requests.clone(),
8604        }));
8605        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8606            base_path: thread_root.clone(),
8607        }));
8608        builder.tool_contributor(Arc::new(ProfileToolContributor));
8609        let runtime =
8610            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8611
8612        let request = captured_thread_override_request(
8613            &runtime,
8614            &requests,
8615            Vec::new(),
8616            Some("You are embedded in a host app.".to_string()),
8617            Vec::new(),
8618        )
8619        .await;
8620
8621        let system = request.instructions.system.expect("system instructions");
8622        assert!(system.starts_with("You are Roder"));
8623        let developer = request
8624            .instructions
8625            .developer
8626            .expect("developer instructions");
8627        assert!(developer.starts_with("You are embedded in a host app."));
8628
8629        let plain =
8630            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8631                .await;
8632        assert_eq!(plain.instructions.developer, None);
8633
8634        let _ = std::fs::remove_dir_all(thread_root);
8635    }
8636
8637    #[tokio::test]
8638    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8639        let requests = Arc::new(StdMutex::new(Vec::new()));
8640        let thread_root = std::env::temp_dir().join(format!(
8641            "roder-thread-external-tools-{}",
8642            uuid::Uuid::new_v4()
8643        ));
8644        let mut builder = ExtensionRegistryBuilder::new();
8645        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8646            requests: requests.clone(),
8647        }));
8648        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8649            base_path: thread_root.clone(),
8650        }));
8651        builder.tool_contributor(Arc::new(ProfileToolContributor));
8652        let runtime =
8653            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8654
8655        let external_tools = vec![
8656            ToolSpec {
8657                name: "acme_lookup".to_string(),
8658                description: "Look up Acme workspace state.".to_string(),
8659                parameters: serde_json::json!({
8660                    "type": "object",
8661                    "properties": { "query": { "type": "string" } },
8662                    "required": ["query"]
8663                }),
8664            },
8665            ToolSpec {
8666                name: "edit".to_string(),
8667                description: "Host-managed edit.".to_string(),
8668                parameters: serde_json::json!({ "type": "object" }),
8669            },
8670        ];
8671        let request =
8672            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8673                .await;
8674
8675        let acme = request
8676            .tools
8677            .iter()
8678            .find(|tool| tool.name == "acme_lookup")
8679            .expect("external tool advertised");
8680        assert_eq!(acme.description, "Look up Acme workspace state.");
8681        assert_eq!(acme.parameters["required"][0], "query");
8682        let edits = request
8683            .tools
8684            .iter()
8685            .filter(|tool| tool.name == "edit")
8686            .collect::<Vec<_>>();
8687        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8688        assert_eq!(edits[0].description, "Host-managed edit.");
8689
8690        let plain =
8691            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8692                .await;
8693        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8694        let plain_edit = plain
8695            .tools
8696            .iter()
8697            .find(|tool| tool.name == "edit")
8698            .expect("builtin edit advertised on plain thread");
8699        assert_eq!(plain_edit.description, "edit test tool");
8700
8701        let _ = std::fs::remove_dir_all(thread_root);
8702    }
8703
8704    #[tokio::test]
8705    async fn model_switch_injects_summary_and_records_profile_segments() {
8706        let requests = Arc::new(StdMutex::new(Vec::new()));
8707        let thread_root = std::env::temp_dir().join(format!(
8708            "roder-model-switch-thread-{}",
8709            uuid::Uuid::new_v4()
8710        ));
8711        let mut builder = ExtensionRegistryBuilder::new();
8712        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8713            requests: requests.clone(),
8714        }));
8715        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8716            base_path: thread_root.clone(),
8717        }));
8718        builder.tool_contributor(Arc::new(ProfileToolContributor));
8719        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8720        claude_profile.provider_family = ProviderFamily::Anthropic;
8721        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8722        let runtime = Arc::new(
8723            Runtime::new(
8724                builder.build().unwrap(),
8725                RuntimeConfig {
8726                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8727                    default_model: "gpt-5.5".to_string(),
8728                    model_profiles: std::collections::HashMap::from([
8729                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8730                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8731                    ]),
8732                    ..RuntimeConfig::default()
8733                },
8734            )
8735            .unwrap(),
8736        );
8737        let thread_id = runtime
8738            .create_thread_with(CreateThreadRequest {
8739                title: Some("Model switch".to_string()),
8740                workspace: test_workspace(),
8741                workspace_id: None,
8742                root_id: None,
8743                provider: None,
8744                model: None,
8745                selection_mode: None,
8746                tool_allowlist: Vec::new(),
8747                developer_instructions: None,
8748                external_tools: Vec::new(),
8749                runner: None,
8750            })
8751            .await
8752            .unwrap()
8753            .thread_id;
8754        let mut rx = runtime.subscribe_events();
8755        for (message, model_override) in [
8756            ("first turn", None),
8757            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8758        ] {
8759            let turn_id = runtime
8760                .start_turn(StartTurnRequest {
8761                    thread_id: thread_id.clone(),
8762                    message: message.to_string(),
8763                    images: Vec::new(),
8764                    provider_override: None,
8765                    model_override,
8766                    reasoning_override: None,
8767                    workspace: test_workspace(),
8768                    instructions: InstructionBundle::default(),
8769                    developer_context: None,
8770                    task_ledger_required: false,
8771                })
8772                .await
8773                .unwrap();
8774            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8775                loop {
8776                    let envelope = rx.recv().await.unwrap();
8777                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8778                        continue;
8779                    }
8780                    match envelope.event {
8781                        RoderEvent::TurnCompleted(_) => break,
8782                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8783                        _ => {}
8784                    }
8785                }
8786            })
8787            .await
8788            .unwrap();
8789        }
8790
8791        let captured = requests.lock().unwrap().clone();
8792        assert_eq!(captured.len(), 2);
8793        assert!(captured[1].transcript.iter().any(|item| {
8794            matches!(
8795                item,
8796                TranscriptItem::UserMessage(message)
8797                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
8798                        && message.text.contains("previous profile mock/gpt-5.5")
8799                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
8800                        && message.text.contains("Available tools now:")
8801            )
8802        }));
8803
8804        let snapshot = runtime
8805            .thread_store
8806            .as_ref()
8807            .unwrap()
8808            .load_thread(&thread_id)
8809            .await
8810            .unwrap()
8811            .unwrap();
8812        let trace_segments = snapshot
8813            .turns
8814            .iter()
8815            .flat_map(|turn| &turn.items)
8816            .filter(|item| {
8817                matches!(
8818                    item,
8819                    TranscriptItem::ProviderMetadata(value)
8820                        if value.get("kind").and_then(serde_json::Value::as_str)
8821                            == Some(MODEL_PROFILE_TRACE_KIND)
8822                            && value.get("segment").and_then(serde_json::Value::as_str)
8823                                == Some("assistant")
8824                )
8825            })
8826            .count();
8827        assert!(trace_segments >= 2);
8828        let _ = std::fs::remove_dir_all(thread_root);
8829    }
8830
8831    struct CountingTaskTool {
8832        calls: Arc<StdMutex<u32>>,
8833    }
8834
8835    #[async_trait::async_trait]
8836    impl ToolExecutor for CountingTaskTool {
8837        fn spec(&self) -> ToolSpec {
8838            ToolSpec {
8839                name: "task".to_string(),
8840                description: "Dispatch a test subagent.".to_string(),
8841                parameters: serde_json::json!({
8842                    "type": "object",
8843                    "properties": {
8844                        "description": { "type": "string" },
8845                        "prompt": { "type": "string" },
8846                        "parent_deadline_seconds": { "type": "integer" }
8847                    },
8848                    "required": ["description", "prompt"],
8849                    "additionalProperties": false
8850                }),
8851            }
8852        }
8853
8854        async fn execute(
8855            &self,
8856            _ctx: ToolExecutionContext,
8857            call: ToolCall,
8858        ) -> anyhow::Result<ToolResult> {
8859            *self.calls.lock().unwrap() += 1;
8860            Ok(ToolResult {
8861                id: call.id,
8862                name: call.name,
8863                text: "started child".to_string(),
8864                data: serde_json::json!({}),
8865                is_error: false,
8866            })
8867        }
8868    }
8869
8870    #[tokio::test]
8871    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
8872        let captured = Arc::new(StdMutex::new(None));
8873        let mut builder = ExtensionRegistryBuilder::new();
8874        builder.inference_engine(Arc::new(CapturingEngine {
8875            request: captured.clone(),
8876        }));
8877        let runtime = Arc::new(
8878            Runtime::new(
8879                builder.build().unwrap(),
8880                RuntimeConfig {
8881                    runtime_profile: RuntimeProfile::NonInteractive,
8882                    ..RuntimeConfig::default()
8883                },
8884            )
8885            .unwrap(),
8886        );
8887        let mut rx = runtime.subscribe_events();
8888        let turn_id = runtime
8889            .start_turn(StartTurnRequest {
8890                thread_id: "thread-profile".to_string(),
8891                message: "work unattended".to_string(),
8892                images: Vec::new(),
8893                provider_override: None,
8894                model_override: None,
8895                reasoning_override: None,
8896                workspace: test_workspace(),
8897                instructions: InstructionBundle {
8898                    system: None,
8899                    developer: Some("base developer".to_string()),
8900                    developer_context: None,
8901                },
8902                developer_context: None,
8903                task_ledger_required: false,
8904            })
8905            .await
8906            .unwrap();
8907
8908        let mut observed_profile = None;
8909        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8910            loop {
8911                let envelope = rx.recv().await.unwrap();
8912                if envelope.turn_id.as_deref() != Some(&turn_id) {
8913                    continue;
8914                }
8915                match envelope.event {
8916                    RoderEvent::TurnStarted(event) => {
8917                        observed_profile = Some(event.runtime_profile);
8918                    }
8919                    RoderEvent::TurnCompleted(_) => break,
8920                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8921                    _ => {}
8922                }
8923            }
8924        })
8925        .await
8926        .unwrap();
8927
8928        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
8929        let request = captured.lock().unwrap().clone().unwrap();
8930        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
8931        let developer = request.instructions.developer.unwrap();
8932        assert!(developer.contains("base developer"));
8933        assert!(developer.contains("non-interactive profile"));
8934    }
8935
8936    #[tokio::test]
8937    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
8938        let workspace = runtime_test_workspace("global-policy-mode");
8939        let thread_root = workspace.join("threads");
8940        let mut builder = ExtensionRegistryBuilder::new();
8941        builder.inference_engine(Arc::new(FakeInferenceEngine));
8942        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8943            base_path: thread_root.clone(),
8944        }));
8945        let runtime = Runtime::new(
8946            builder.build().unwrap(),
8947            RuntimeConfig {
8948                workspace: Some(workspace.display().to_string()),
8949                ..Default::default()
8950            },
8951        )
8952        .unwrap();
8953
8954        runtime
8955            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
8956            .await
8957            .unwrap();
8958
8959        assert!(!thread_root.join("runtime").exists());
8960        let _ = std::fs::remove_dir_all(workspace);
8961    }
8962
8963    #[tokio::test]
8964    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
8965        let captured = Arc::new(StdMutex::new(None));
8966        let mut builder = ExtensionRegistryBuilder::new();
8967        builder.inference_engine(Arc::new(CapturingEngine {
8968            request: captured.clone(),
8969        }));
8970        builder.tool_contributor(Arc::new(
8971            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
8972        ));
8973        let runtime = Arc::new(
8974            Runtime::new(
8975                builder.build().unwrap(),
8976                RuntimeConfig {
8977                    runtime_profile: RuntimeProfile::Eval,
8978                    policy_mode: PolicyMode::Bypass,
8979                    agent_swarm_mode: false,
8980                    ..RuntimeConfig::default()
8981                },
8982            )
8983            .unwrap(),
8984        );
8985        let mut rx = runtime.subscribe_events();
8986        let turn_id = runtime
8987            .start_turn(StartTurnRequest {
8988                thread_id: "thread-ledger".to_string(),
8989                message: "decomposed work".to_string(),
8990                images: Vec::new(),
8991                provider_override: None,
8992                model_override: None,
8993                reasoning_override: None,
8994                workspace: test_workspace(),
8995                instructions: InstructionBundle::default(),
8996                developer_context: None,
8997                task_ledger_required: true,
8998            })
8999            .await
9000            .unwrap();
9001
9002        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9003            loop {
9004                let envelope = rx.recv().await.unwrap();
9005                if envelope.turn_id.as_deref() == Some(&turn_id)
9006                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9007                {
9008                    break;
9009                }
9010            }
9011        })
9012        .await
9013        .unwrap();
9014
9015        let request = captured.lock().unwrap().clone().unwrap();
9016        let developer = request.instructions.developer.unwrap();
9017        assert!(developer.contains("Task Ledger Required"));
9018        assert!(developer.contains("task_ledger.update"));
9019        let tool_names: Vec<_> = request
9020            .tools
9021            .iter()
9022            .map(|tool| tool.name.as_str())
9023            .collect();
9024        assert!(
9025            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9026            "tool names: {tool_names:?}"
9027        );
9028        assert_eq!(
9029            request.tool_choice,
9030            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9031        );
9032        assert_eq!(request.tools.len(), 1);
9033        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9034    }
9035
9036    #[tokio::test]
9037    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9038        let requests = Arc::new(StdMutex::new(Vec::new()));
9039        let mut builder = ExtensionRegistryBuilder::new();
9040        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9041            calls: StdMutex::new(0),
9042            requests: requests.clone(),
9043        }));
9044        builder.tool_contributor(Arc::new(
9045            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9046        ));
9047        let runtime = Arc::new(
9048            Runtime::new(
9049                builder.build().unwrap(),
9050                RuntimeConfig {
9051                    runtime_profile: RuntimeProfile::Eval,
9052                    policy_mode: PolicyMode::Bypass,
9053                    agent_swarm_mode: false,
9054                    ..RuntimeConfig::default()
9055                },
9056            )
9057            .unwrap(),
9058        );
9059        let mut rx = runtime.subscribe_events();
9060        let turn_id = runtime
9061            .start_turn(StartTurnRequest {
9062                thread_id: "thread-ledger-completion".to_string(),
9063                message: "write the answer file".to_string(),
9064                images: Vec::new(),
9065                provider_override: None,
9066                model_override: None,
9067                reasoning_override: None,
9068                workspace: test_workspace(),
9069                instructions: InstructionBundle::default(),
9070                developer_context: None,
9071                task_ledger_required: true,
9072            })
9073            .await
9074            .unwrap();
9075
9076        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9077            loop {
9078                let envelope = rx.recv().await.unwrap();
9079                if envelope.turn_id.as_deref() != Some(&turn_id) {
9080                    continue;
9081                }
9082                match envelope.event {
9083                    RoderEvent::TurnCompleted(_) => break,
9084                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9085                    _ => {}
9086                }
9087            }
9088        })
9089        .await
9090        .unwrap();
9091
9092        let requests = requests.lock().unwrap().clone();
9093        assert_eq!(requests.len(), 4);
9094        assert!(requests[2].transcript.iter().any(|item| {
9095            matches!(
9096                item,
9097                TranscriptItem::UserMessage(message)
9098                    if message.text.contains("Task Ledger Completion Required")
9099                        && message.text.contains("Write /app/result.txt")
9100            )
9101        }));
9102        assert!(requests[3].transcript.iter().any(|item| {
9103            matches!(
9104                item,
9105                TranscriptItem::ToolResult(result)
9106                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9107                        && result.result.contains("Task ledger: 2/2 completed")
9108            )
9109        }));
9110    }
9111
9112    #[tokio::test]
9113    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9114        let requests = Arc::new(StdMutex::new(Vec::new()));
9115        let mut builder = ExtensionRegistryBuilder::new();
9116        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9117            calls: StdMutex::new(0),
9118            requests: requests.clone(),
9119        }));
9120        builder.tool_contributor(Arc::new(
9121            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9122        ));
9123        let runtime = Arc::new(
9124            Runtime::new(
9125                builder.build().unwrap(),
9126                RuntimeConfig {
9127                    runtime_profile: RuntimeProfile::Eval,
9128                    policy_mode: PolicyMode::Bypass,
9129                    agent_swarm_mode: false,
9130                    turn_deadline_seconds: Some(120),
9131                    ..RuntimeConfig::default()
9132                },
9133            )
9134            .unwrap(),
9135        );
9136        let mut rx = runtime.subscribe_events();
9137        let turn_id = runtime
9138            .start_turn(StartTurnRequest {
9139                thread_id: "thread-ledger-checkpoint".to_string(),
9140                message: "write the answer file".to_string(),
9141                images: Vec::new(),
9142                provider_override: None,
9143                model_override: None,
9144                reasoning_override: None,
9145                workspace: test_workspace(),
9146                instructions: InstructionBundle::default(),
9147                developer_context: None,
9148                task_ledger_required: true,
9149            })
9150            .await
9151            .unwrap();
9152
9153        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9154            loop {
9155                let envelope = rx.recv().await.unwrap();
9156                if envelope.turn_id.as_deref() != Some(&turn_id) {
9157                    continue;
9158                }
9159                match envelope.event {
9160                    RoderEvent::TurnCompleted(_) => break,
9161                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9162                    _ => {}
9163                }
9164            }
9165        })
9166        .await
9167        .unwrap();
9168
9169        let requests = requests.lock().unwrap().clone();
9170        assert!(requests.len() >= 2);
9171        assert!(requests[1].transcript.iter().any(|item| {
9172            matches!(
9173                item,
9174                TranscriptItem::UserMessage(message)
9175                    if message.text.contains("Scoreable Output Checkpoint")
9176                        && message.text.contains("ensure the required output file(s) exist")
9177                        && message.text.contains("Write /app/result.txt")
9178            )
9179        }));
9180    }
9181
9182    #[test]
9183    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9184        let prompt = task_ledger_deadline_completion_prompt(
9185            12,
9186            30,
9187            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9188        );
9189
9190        assert!(prompt.contains("12 seconds remain"));
9191        assert!(prompt.contains("create or update the required scoreable output files"));
9192        assert!(prompt.contains("write /app/result.txt"));
9193        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9194    }
9195
9196    #[test]
9197    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9198        let prompt = task_ledger_scoreable_checkpoint_prompt(
9199            120,
9200            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9201        );
9202
9203        assert!(prompt.contains("120 seconds remain"));
9204        assert!(prompt.contains("best evidence-backed answer"));
9205        assert!(prompt.contains("even if provisional"));
9206        assert!(prompt.contains("preserve that candidate"));
9207        assert!(prompt.contains("partial-coverage"));
9208        assert!(prompt.contains("write /app/result.txt"));
9209        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9210    }
9211
9212    #[test]
9213    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9214        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9215        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9216            id: "ledger-open".to_string(),
9217            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9218            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9219                .to_string(),
9220            display_payload: None,
9221            is_error: false,
9222        })];
9223
9224        let (_, action) = inference_timeout_deadline(
9225            deadline,
9226            RuntimeProfile::Eval,
9227            true,
9228            30,
9229            false,
9230            0,
9231            &transcript,
9232        )
9233        .unwrap();
9234
9235        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9236    }
9237
9238    #[tokio::test]
9239    async fn verification_gate_forces_eval_code_changes_through_review() {
9240        let mut builder = ExtensionRegistryBuilder::new();
9241        builder.inference_engine(Arc::new(VerificationGateEngine {
9242            calls: StdMutex::new(0),
9243        }));
9244        builder.tool_contributor(Arc::new(WriteFileContributor));
9245        builder.tool_contributor(Arc::new(
9246            roder_ext_verification::VerificationToolContributor,
9247        ));
9248        let runtime = Arc::new(
9249            Runtime::new(
9250                builder.build().unwrap(),
9251                RuntimeConfig {
9252                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9253                    default_model: "mock".to_string(),
9254                    runtime_profile: RuntimeProfile::Eval,
9255                    policy_mode: PolicyMode::Bypass,
9256                    agent_swarm_mode: false,
9257                    ..RuntimeConfig::default()
9258                },
9259            )
9260            .unwrap(),
9261        );
9262        let mut rx = runtime.subscribe_events();
9263        let turn_id = runtime
9264            .start_turn(StartTurnRequest {
9265                thread_id: "thread-verification".to_string(),
9266                message: "write code".to_string(),
9267                images: Vec::new(),
9268                provider_override: None,
9269                model_override: None,
9270                reasoning_override: None,
9271                workspace: test_workspace(),
9272                instructions: InstructionBundle::default(),
9273                developer_context: None,
9274                task_ledger_required: false,
9275            })
9276            .await
9277            .unwrap();
9278
9279        let mut saw_required = false;
9280        let mut saw_completed = false;
9281        let mut final_text = String::new();
9282        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9283            loop {
9284                let envelope = rx.recv().await.unwrap();
9285                if envelope.turn_id.as_deref() != Some(&turn_id) {
9286                    continue;
9287                }
9288                match envelope.event {
9289                    RoderEvent::VerificationRequired(event) => {
9290                        saw_required = true;
9291                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9292                    }
9293                    RoderEvent::VerificationCompleted(event) => {
9294                        saw_completed = true;
9295                        assert!(event.passed);
9296                    }
9297                    RoderEvent::InferenceEventReceived(event) => {
9298                        if let InferenceEvent::MessageDelta(delta) = event.event {
9299                            final_text.push_str(&delta.text);
9300                        }
9301                    }
9302                    RoderEvent::TurnCompleted(_) => break,
9303                    _ => {}
9304                }
9305            }
9306        })
9307        .await
9308        .unwrap();
9309
9310        assert!(saw_required);
9311        assert!(saw_completed);
9312        assert!(final_text.contains("verified final"));
9313    }
9314
9315    #[tokio::test]
9316    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9317        let requests = Arc::new(StdMutex::new(Vec::new()));
9318        let mut builder = ExtensionRegistryBuilder::new();
9319        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9320            calls: StdMutex::new(0),
9321            requests: requests.clone(),
9322        }));
9323        builder.tool_contributor(Arc::new(WriteFileContributor));
9324        builder.tool_contributor(Arc::new(
9325            roder_ext_verification::VerificationToolContributor,
9326        ));
9327        let runtime = Arc::new(
9328            Runtime::new(
9329                builder.build().unwrap(),
9330                RuntimeConfig {
9331                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9332                    default_model: "gpt-5.5".to_string(),
9333                    runtime_profile: RuntimeProfile::Eval,
9334                    policy_mode: PolicyMode::Bypass,
9335                    agent_swarm_mode: false,
9336                    ..RuntimeConfig::default()
9337                },
9338            )
9339            .unwrap(),
9340        );
9341        let mut rx = runtime.subscribe_events();
9342        let turn_id = runtime
9343            .start_turn(StartTurnRequest {
9344                thread_id: "thread-speed-policy".to_string(),
9345                message: "write code".to_string(),
9346                images: Vec::new(),
9347                provider_override: None,
9348                model_override: None,
9349                reasoning_override: None,
9350                workspace: test_workspace(),
9351                instructions: InstructionBundle::default(),
9352                developer_context: None,
9353                task_ledger_required: false,
9354            })
9355            .await
9356            .unwrap();
9357
9358        let mut saw_speed_policy_event = false;
9359        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9360            loop {
9361                let envelope = rx.recv().await.unwrap();
9362                if envelope.turn_id.as_deref() != Some(&turn_id) {
9363                    continue;
9364                }
9365                match envelope.event {
9366                    RoderEvent::InferenceStarted(event) => {
9367                        if event.speed_policy.is_some() {
9368                            saw_speed_policy_event = true;
9369                        }
9370                    }
9371                    RoderEvent::TurnCompleted(_) => break,
9372                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9373                    _ => {}
9374                }
9375            }
9376        })
9377        .await
9378        .unwrap();
9379
9380        let requests = requests.lock().unwrap().clone();
9381        assert!(saw_speed_policy_event);
9382        assert!(requests.len() >= 4);
9383        assert!(requests.iter().all(|request| {
9384            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9385                && request.model.model == "gpt-5.5"
9386        }));
9387        assert_eq!(
9388            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9389            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9390        );
9391        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9392        assert_eq!(
9393            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9394            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9395        );
9396        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9397        assert_eq!(
9398            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9399            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9400        );
9401        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9402        assert_eq!(
9403            requests[2]
9404                .metadata
9405                .pointer("/speedPolicy/phase")
9406                .and_then(serde_json::Value::as_str),
9407            Some("verification")
9408        );
9409    }
9410
9411    #[tokio::test]
9412    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9413        let mut builder = ExtensionRegistryBuilder::new();
9414        builder.inference_engine(Arc::new(DeadlineEngine));
9415        let runtime = Arc::new(
9416            Runtime::new(
9417                builder.build().unwrap(),
9418                RuntimeConfig {
9419                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9420                    default_model: "mock".to_string(),
9421                    runtime_profile: RuntimeProfile::Eval,
9422                    turn_deadline_seconds: Some(1),
9423                    ..RuntimeConfig::default()
9424                },
9425            )
9426            .unwrap(),
9427        );
9428        let mut rx = runtime.subscribe_events();
9429        let turn_id = runtime
9430            .start_turn(StartTurnRequest {
9431                thread_id: "thread-deadline".to_string(),
9432                message: "slow work".to_string(),
9433                images: Vec::new(),
9434                provider_override: None,
9435                model_override: None,
9436                reasoning_override: None,
9437                workspace: test_workspace(),
9438                instructions: InstructionBundle::default(),
9439                developer_context: None,
9440                task_ledger_required: false,
9441            })
9442            .await
9443            .unwrap();
9444
9445        let mut saw_partial = false;
9446        let mut saw_deadline = false;
9447        let mut failed_kind = None;
9448        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9449            loop {
9450                let envelope = rx.recv().await.unwrap();
9451                if envelope.turn_id.as_deref() != Some(&turn_id) {
9452                    continue;
9453                }
9454                match envelope.event {
9455                    RoderEvent::TurnPartialResult(event) => {
9456                        saw_partial = event.summary.contains("partial turn state");
9457                    }
9458                    RoderEvent::TurnDeadlineExceeded(event) => {
9459                        saw_deadline = event.partial_result.contains("transcript items");
9460                    }
9461                    RoderEvent::TurnFailed(event) => {
9462                        failed_kind = event.error_kind;
9463                        break;
9464                    }
9465                    _ => {}
9466                }
9467            }
9468        })
9469        .await
9470        .unwrap();
9471
9472        for _ in 0..20 {
9473            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9474                break;
9475            }
9476            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9477        }
9478        assert!(saw_partial);
9479        assert!(saw_deadline);
9480        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9481        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9482    }
9483
9484    #[tokio::test]
9485    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9486        let calls = Arc::new(StdMutex::new(0));
9487        let mut builder = ExtensionRegistryBuilder::new();
9488        builder.inference_engine(Arc::new(CapturingEngine {
9489            request: Arc::new(StdMutex::new(None)),
9490        }));
9491        let task_tool = Arc::new(CountingTaskTool {
9492            calls: calls.clone(),
9493        });
9494        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9495        let runtime = Arc::new(
9496            Runtime::new(
9497                builder.build().unwrap(),
9498                RuntimeConfig {
9499                    policy_mode: PolicyMode::Bypass,
9500                    agent_swarm_mode: false,
9501                    ..RuntimeConfig::default()
9502                },
9503            )
9504            .unwrap(),
9505        );
9506
9507        let result = runtime
9508            .route_tool_call(
9509                &"thread-deadline-task".to_string(),
9510                &"turn-deadline-task".to_string(),
9511                ToolCallCompleted {
9512                    id: "task-1".to_string(),
9513                    name: "task".to_string(),
9514                    arguments: serde_json::json!({
9515                        "description": "inspect",
9516                        "prompt": "read"
9517                    })
9518                    .to_string(),
9519                },
9520                None,
9521                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9522            )
9523            .await
9524            .unwrap();
9525
9526        assert!(result.is_error);
9527        assert!(result.result.contains("deadline policy skipped"));
9528        assert_eq!(*calls.lock().unwrap(), 0);
9529    }
9530
9531    struct TestToolContributor {
9532        tool: Arc<dyn ToolExecutor>,
9533    }
9534
9535    impl ToolContributor for TestToolContributor {
9536        fn id(&self) -> String {
9537            "test-tool".to_string()
9538        }
9539
9540        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9541            registry.register(self.tool.clone())
9542        }
9543    }
9544
9545    #[tokio::test]
9546    async fn agent_swarm_mode_override_is_per_thread() {
9547        let runtime = Runtime::fake().unwrap();
9548        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9549
9550        // Off everywhere by default.
9551        assert!(
9552            !runtime
9553                .effective_agent_swarm_mode_for_thread("thread-a")
9554                .await
9555        );
9556        assert!(
9557            !runtime
9558                .effective_agent_swarm_mode_for_thread("thread-b")
9559                .await
9560        );
9561
9562        // Enabling on thread-a does not leak into thread-b.
9563        assert!(
9564            runtime
9565                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9566                .await
9567        );
9568        assert!(
9569            runtime
9570                .effective_agent_swarm_mode_for_thread("thread-a")
9571                .await
9572        );
9573        assert!(
9574            !runtime
9575                .effective_agent_swarm_mode_for_thread("thread-b")
9576                .await
9577        );
9578
9579        // A per-thread `off` override wins over a runtime-global `on` default.
9580        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9581        runtime
9582            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9583            .await;
9584        assert!(
9585            !runtime
9586                .effective_agent_swarm_mode_for_thread("thread-b")
9587                .await,
9588            "explicit per-thread off overrides the global on default"
9589        );
9590        // A thread with no override still follows the global default.
9591        assert!(
9592            runtime
9593                .effective_agent_swarm_mode_for_thread("thread-c")
9594                .await,
9595            "threads without an override follow the runtime-global default"
9596        );
9597    }
9598
9599    #[tokio::test]
9600    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9601        let runtime = Runtime::fake().unwrap();
9602        let mut events = runtime.subscribe_events();
9603        runtime
9604            .set_agent_swarm_mode_for_thread(
9605                "thread-xyz",
9606                true,
9607                roder_api::subagents::AgentSwarmModeTrigger::Task,
9608            )
9609            .await;
9610        let mut saw = false;
9611        for _ in 0..8 {
9612            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9613                .await
9614                .unwrap()
9615                .unwrap();
9616            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9617                assert_eq!(event.thread_id, "thread-xyz");
9618                assert!(event.enabled);
9619                assert_eq!(
9620                    event.trigger,
9621                    roder_api::subagents::AgentSwarmModeTrigger::Task
9622                );
9623                saw = true;
9624                break;
9625            }
9626        }
9627        assert!(
9628            saw,
9629            "expected an AgentSwarmModeChanged event for the thread"
9630        );
9631    }
9632}