Skip to main content

roder_core/
runtime.rs

1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::Arc;
4use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12    model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33    ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37    RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42    TeamMemberStatus,
43};
44use roder_api::thread::{
45    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51    TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67    classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73    route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77    apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78    apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83    provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
94/// limit into a continuation, or as the empty-tool-call persistence nudge, so
95/// the model keeps working instead of ending the turn early.
96const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
98/// its render loop, which during an active turn only wakes at the ~6 FPS status
99/// animation cadence (backend events do not wake the input poll), so up to
100/// ~166ms of events buffer between drains. A reasoning-high provider that runs
101/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
102/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
103/// 1024-slot buffer overflowed in those windows and silently dropped tool and
104/// thinking rows from the live view. Sized for generous headroom across bursts
105/// and brief render stalls.
106const EVENT_BUS_CAPACITY: usize = 16_384;
107const FINAL_ANSWER_PHASE: &str = "final_answer";
108pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
109const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
110const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
111const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
112pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
113const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
114const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
115const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
116
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum InferenceTimeoutAction {
119    ScoreableCheckpoint,
120    Finalization,
121}
122
123#[derive(Debug, Clone)]
124pub struct RuntimeConfig {
125    pub default_provider: String,
126    pub default_model: String,
127    pub reasoning: Option<String>,
128    pub auto_compact_token_limit: Option<u32>,
129    pub file_backed_dynamic_context: bool,
130    pub hosted_web_search: HostedWebSearchConfig,
131    pub tool_search: ToolSearchConfig,
132    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
133    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
134    pub model_edit_tools: HashMap<String, String>,
135    pub model_parallel_tool_calls: HashMap<String, bool>,
136    pub model_profiles: HashMap<String, ModelHarnessProfile>,
137    pub tool_allowlist: Vec<String>,
138    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
139    pub external_tool_timeout_seconds: u64,
140    pub command_shell: String,
141    pub workspace: Option<String>,
142    pub policy_mode: PolicyMode,
143    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
144    /// injects the swarm reminder into each turn's developer instructions so any
145    /// client benefits, not just the TUI.
146    pub agent_swarm_mode: bool,
147    pub runtime_profile: RuntimeProfile,
148    pub inference_router: RuntimeInferenceRouterConfig,
149    pub speed_policy: RuntimeSpeedPolicyConfig,
150    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
151    pub reliability: RuntimeReliabilityConfig,
152    pub turn_deadline_seconds: Option<u64>,
153    pub remote_runner_destination: Option<RunnerDestination>,
154    pub team_data_dir: Option<PathBuf>,
155    pub roadmap_data_dir: Option<PathBuf>,
156    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
157}
158
159impl Default for RuntimeConfig {
160    fn default() -> Self {
161        Self {
162            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
163            default_model: "mock".to_string(),
164            reasoning: None,
165            auto_compact_token_limit: None,
166            file_backed_dynamic_context: true,
167            hosted_web_search: HostedWebSearchConfig::cached(),
168            tool_search: ToolSearchConfig::default(),
169            provider_tool_search: HashMap::new(),
170            model_tool_search: HashMap::new(),
171            model_edit_tools: HashMap::new(),
172            model_parallel_tool_calls: HashMap::new(),
173            model_profiles: HashMap::new(),
174            tool_allowlist: Vec::new(),
175            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
176            command_shell: roder_api::command_shell::default_command_shell(),
177            workspace: None,
178            policy_mode: PolicyMode::Default,
179            agent_swarm_mode: false,
180            runtime_profile: RuntimeProfile::Interactive,
181            inference_router: RuntimeInferenceRouterConfig::default(),
182            speed_policy: RuntimeSpeedPolicyConfig::default(),
183            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
184            reliability: RuntimeReliabilityConfig::default(),
185            turn_deadline_seconds: None,
186            remote_runner_destination: None,
187            team_data_dir: None,
188            roadmap_data_dir: None,
189            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
190        }
191    }
192}
193
194#[derive(Debug, Clone)]
195pub struct StartTurnRequest {
196    pub thread_id: ThreadId,
197    pub message: String,
198    pub images: Vec<InputImage>,
199    pub provider_override: Option<String>,
200    pub model_override: Option<String>,
201    pub reasoning_override: Option<String>,
202    pub workspace: String,
203    pub instructions: InstructionBundle,
204    /**
205     * Per-turn developer-authority context for this turn's InstructionBundle.
206     * Applies to every inference round of the turn, is never written to
207     * thread state, and does not carry over to later turns.
208     */
209    pub developer_context: Option<String>,
210    pub task_ledger_required: bool,
211}
212
213#[derive(Debug, Clone)]
214pub struct CreateThreadRequest {
215    pub title: Option<String>,
216    pub workspace: String,
217    pub workspace_id: Option<String>,
218    pub root_id: Option<String>,
219    pub provider: Option<String>,
220    pub model: Option<String>,
221    pub selection_mode: Option<ModelSelectionMode>,
222    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
223    pub tool_allowlist: Vec<String>,
224    /// Host-supplied instructions added to the developer slot of every turn's inference request.
225    pub developer_instructions: Option<String>,
226    /// Host-executed tool specs advertised to the model on every turn of this thread.
227    pub external_tools: Vec<roder_api::tools::ToolSpec>,
228    /// Explicit remote-runner binding for the thread's native coding tools.
229    pub runner: Option<ThreadRunnerSelection>,
230}
231
232/**
233 * Thread-level remote-runner selection. The destination config is persisted
234 * with the thread, so secrets must reach the provider through its
235 * environment, not this config.
236 */
237#[derive(Debug, Clone)]
238pub struct ThreadRunnerSelection {
239    pub provider_id: String,
240    pub config: serde_json::Value,
241    /// Absolute path on the runner used as the thread's coding-tool workspace root.
242    pub workspace: String,
243    /**
244     * Extra absolute runner paths file reads may resolve under, beyond
245     * `workspace`. Writes and the working directory stay confined to
246     * `workspace`.
247     */
248    pub read_roots: Vec<String>,
249}
250
251#[derive(Debug, Clone, PartialEq, Eq)]
252pub struct PendingPlanExit {
253    pub thread_id: ThreadId,
254    pub turn_id: TurnId,
255    pub request_id: String,
256    pub target_mode: PolicyMode,
257    pub plan_summary: Option<String>,
258    pub next_steps: Vec<String>,
259    pub requested_at: OffsetDateTime,
260    pub expires_at: Option<OffsetDateTime>,
261}
262
263pub(crate) struct PendingToolApproval {
264    pub(crate) thread_id: ThreadId,
265    pub(crate) turn_id: TurnId,
266    pub(crate) tool_id: String,
267    pub(crate) tool_name: String,
268    pub(crate) call: roder_api::tools::ToolCall,
269    pub(crate) tx: oneshot::Sender<bool>,
270}
271
272pub(crate) struct PendingUserInput {
273    pub(crate) thread_id: ThreadId,
274    pub(crate) turn_id: TurnId,
275    pub(crate) tx: oneshot::Sender<serde_json::Value>,
276}
277
278/// Host answer to an external tool call delivered via `tools/resolve`.
279#[derive(Debug, Clone, PartialEq, Eq)]
280pub struct ExternalToolResolution {
281    pub output: String,
282    pub is_error: bool,
283}
284
285pub(crate) struct PendingExternalToolCall {
286    pub(crate) thread_id: ThreadId,
287    pub(crate) turn_id: TurnId,
288    pub(crate) tool_id: String,
289    pub(crate) tool_name: String,
290    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
291}
292
293#[derive(Clone)]
294struct ActiveTurnHandle {
295    thread_id: ThreadId,
296    abort: AbortHandle,
297    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
298    drain: Arc<TurnDrainHandle>,
299}
300
301/// Tracks a task after it has been interrupted and removed from the interactive
302/// active-turn map. This lets users start a follow-up turn immediately while a
303/// bounded runtime shutdown can still await the original task's cleanup.
304struct TurnDrainHandle {
305    thread_id: ThreadId,
306    interrupt_requested: AtomicBool,
307    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
308    completed: AtomicBool,
309    completed_notify: Notify,
310}
311
312/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
313/// tasks have reached their cleanup paths; provider-specific child-process
314/// reaping remains an explicit provider capability.
315#[derive(Debug, Clone, PartialEq, Eq)]
316pub enum RuntimeDrainOutcome {
317    Clean {
318        interrupted_turn_ids: Vec<TurnId>,
319    },
320    DeadlineExceeded {
321        interrupted_turn_ids: Vec<TurnId>,
322        remaining_turn_ids: Vec<TurnId>,
323    },
324    /// At least one lifecycle transition initiated by this drain could not be
325    /// durably appended. The runtime still made its best cleanup attempt, but
326    /// callers must not claim a persisted terminal state as proof of recovery.
327    PersistenceFailed {
328        interrupted_turn_ids: Vec<TurnId>,
329        remaining_turn_ids: Vec<TurnId>,
330    },
331}
332
333#[derive(Clone)]
334struct QueuedTurnSteer {
335    message: UserMessage,
336    mailbox_ack: Option<MailboxDeliveryAck>,
337}
338
339#[derive(Clone)]
340struct MailboxDeliveryAck {
341    team_id: TeamId,
342    message_ids: Vec<String>,
343}
344
345#[derive(Clone)]
346struct InheritedTurnContext {
347    workspace: String,
348    instructions: InstructionBundle,
349    developer_context: Option<String>,
350}
351
352#[derive(Debug, Clone, Default, PartialEq, Eq)]
353pub struct ThreadActivity {
354    pub active_turn_id: Option<TurnId>,
355    pub active_flags: Vec<String>,
356}
357
358/// Per-thread settings persisted at thread creation and applied to every turn.
359#[derive(Debug, Clone, Default)]
360pub(crate) struct ThreadTurnOverrides {
361    pub(crate) tool_allowlist: Vec<String>,
362    pub(crate) developer_instructions: Option<String>,
363    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
364}
365
366#[derive(Debug, Clone, Copy, PartialEq, Eq)]
367pub(crate) enum TurnRunOutcome {
368    Completed,
369    Stopped,
370}
371
372impl PendingPlanExit {
373    pub fn new(
374        thread_id: ThreadId,
375        turn_id: TurnId,
376        request_id: String,
377        target_mode: PolicyMode,
378        plan_summary: Option<String>,
379        next_steps: Vec<String>,
380    ) -> Self {
381        let requested_at = OffsetDateTime::now_utc();
382        Self {
383            thread_id,
384            turn_id,
385            request_id,
386            target_mode,
387            plan_summary,
388            next_steps,
389            requested_at,
390            expires_at: Some(requested_at + default_plan_exit_timeout()),
391        }
392    }
393
394    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
395        self.expires_at.is_some_and(|expires_at| now >= expires_at)
396    }
397}
398
399pub fn default_plan_exit_timeout() -> Duration {
400    Duration::minutes(10)
401}
402
403pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
404
405fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
406    messages
407        .iter()
408        .map(|message| {
409            let recipient = team
410                .members
411                .iter()
412                .find(|member| member.id == message.to_member_id)
413                .map(canonical_team_member_path)
414                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
415            let sender = message
416                .from_member_id
417                .as_deref()
418                .and_then(|id| team.members.iter().find(|member| member.id == id))
419                .map(canonical_team_member_path)
420                .unwrap_or_else(|| "/root".to_string());
421            let message_type = match message.kind {
422                TeamMailboxMessageKind::Message => "MESSAGE",
423                TeamMailboxMessageKind::NewTask => "NEW_TASK",
424                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
425            };
426            format!(
427                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
428                message.text
429            )
430        })
431        .collect::<Vec<_>>()
432        .join("\n\n")
433}
434
435fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
436    if let Some(agent_path) = member
437        .agent_path
438        .as_deref()
439        .filter(|path| *path == "/root" || path.starts_with("/root/"))
440    {
441        return agent_path.to_string();
442    }
443    if member.role == TeamMemberRole::Lead {
444        return "/root".to_string();
445    }
446    member
447        .task_name
448        .as_deref()
449        .filter(|name| !name.trim().is_empty())
450        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
451        .unwrap_or_else(|| format!("/root/{}", member.id))
452}
453
454fn is_codex_v2_team(team: &TeamState) -> bool {
455    team.members.iter().any(|member| {
456        member
457            .model
458            .as_deref()
459            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
460    })
461}
462
463fn runtime_local_team_view(
464    mut team: TeamState,
465    active_thread_ids: &std::collections::HashSet<ThreadId>,
466) -> TeamState {
467    for member in &mut team.members {
468        if member.status == TeamMemberStatus::Running
469            && !active_thread_ids.contains(&member.thread_id)
470        {
471            member.status = TeamMemberStatus::Interrupted;
472            member.current_turn_id = None;
473        }
474    }
475    team
476}
477
478pub struct Runtime {
479    pub bus: EventBus,
480    pub registry: ExtensionRegistry,
481    config: RwLock<RuntimeConfig>,
482    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
483    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
484    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
485    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
486    /// Serializes turn admission with shutdown quiescing. A drain takes this
487    /// gate before flipping `accepting_turns` and snapshotting active work so a
488    /// turn that observed the old flag cannot be inserted after the snapshot.
489    turn_admission: Mutex<()>,
490    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
491    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
492    /// Provider cleanup handles register synchronously through the inference
493    /// tool-executor context. They are intentionally separate from active turn
494    /// admission so an interrupted turn can remain drainable after a follow-up
495    /// turn starts on the same thread.
496    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
497    accepting_turns: AtomicBool,
498    /// Monotonic counter used by bounded drains to surface lifecycle-state
499    /// persistence failures without making an individual provider turn fail.
500    lifecycle_persistence_failures: AtomicUsize,
501    lifecycle_shutdown_drains: AtomicUsize,
502    lifecycle_clean_shutdowns: AtomicUsize,
503    lifecycle_deadline_exceeded: AtomicUsize,
504    lifecycle_persistence_failed_drains: AtomicUsize,
505    lifecycle_restart_reconciliations: AtomicUsize,
506    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
507    provider_cleanup_confirmed: AtomicUsize,
508    provider_cleanup_timed_out: AtomicUsize,
509    provider_cleanup_unknown: AtomicUsize,
510    active_turns_changed: Notify,
511    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
512    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
513    /// Process-local execution boundary. Local/CLI runtimes default to true;
514    /// hosted runtime pools set this false so missing runner metadata fails
515    /// closed instead of exposing the host workspace.
516    allow_local_workspaces: AtomicBool,
517    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
518    // This stays process-local because developer_context is explicitly volatile and must never
519    // be persisted to thread state.
520    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
521    workspace: PathBuf,
522    teams: TeamManager,
523    agent_team_spawn_lock: Mutex<()>,
524    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
525    pub(crate) goals: Arc<RuntimeGoalController>,
526    context_artifacts: roder_api::artifacts::ContextArtifactStore,
527    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
528    thread_item_cache: Mutex<ThreadItemCache>,
529    pub(crate) tool_registry: ToolRegistry,
530    media_generation: Arc<crate::media_generation::MediaGenerationService>,
531    pub(crate) skills: RwLock<SkillRegistry>,
532    /// Lazily-started bounded dispatch of emitted events to registry
533    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
534    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
535    pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
536    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
537    /// this map uses its stored value; absent threads fall back to the
538    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
539    /// team per-member policy-mode override idiom so swarm mode is per-thread
540    /// like the other `thread/*` operations, without a separate runtime.
541    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
542    /**
543     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
544     * held across provider initialization, making the first workspace-tool
545     * access a singleflight while allowing unrelated threads to initialize in
546     * parallel. Failed initialization is shared with current waiters, then
547     * evicted so a later tool call can retry.
548     */
549    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
550}
551
552#[derive(Clone)]
553struct CachedRunnerSession {
554    destination: RunnerDestination,
555    session: Arc<dyn RemoteRunnerSession>,
556}
557
558struct RunnerSessionSlot {
559    provider_id: String,
560    destination_id: String,
561    state: Mutex<RunnerSessionSlotState>,
562    initialized: Notify,
563}
564
565enum RunnerSessionSlotState {
566    Empty,
567    Initializing,
568    Ready(CachedRunnerSession),
569    Failed(Arc<str>),
570}
571
572impl RunnerSessionSlot {
573    fn empty(destination: &RunnerDestination) -> Self {
574        Self {
575            provider_id: destination.provider_id.clone(),
576            destination_id: destination.id.clone(),
577            state: Mutex::new(RunnerSessionSlotState::Empty),
578            initialized: Notify::new(),
579        }
580    }
581
582    fn ready(session: CachedRunnerSession) -> Self {
583        Self {
584            provider_id: session.destination.provider_id.clone(),
585            destination_id: session.destination.id.clone(),
586            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
587            initialized: Notify::new(),
588        }
589    }
590
591    fn matches(&self, destination: &RunnerDestination) -> bool {
592        self.provider_id == destination.provider_id && self.destination_id == destination.id
593    }
594}
595
596impl Runtime {
597    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
598        if registry.inference_engines.is_empty() {
599            anyhow::bail!("at least one inference engine must be registered");
600        }
601        validate_runtime_config_reasoning(&config)?;
602        validate_runtime_inference_router_config(&registry, &config)?;
603
604        let bus = EventBus::new(EVENT_BUS_CAPACITY);
605        let thread_store = registry
606            .thread_stores
607            .first()
608            .map(|factory| factory.create());
609        let mut tool_registry = ToolRegistry::default();
610        for contributor in &registry.tools {
611            contributor
612                .contribute(&mut tool_registry)
613                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
614        }
615        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
616
617        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
618            registry.media_generator_providers.clone(),
619            config.media_generation.clone(),
620        ));
621        tool_registry.replace(Arc::new(
622            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
623        ));
624
625        let team_data_dir = config.team_data_dir.clone();
626        let workspace = config
627            .workspace
628            .clone()
629            .map(PathBuf::from)
630            .unwrap_or(std::env::current_dir()?);
631        let roadmap_data_dir = config
632            .roadmap_data_dir
633            .clone()
634            .unwrap_or_else(|| workspace.join(".roder"));
635        let context_artifacts = thread_store
636            .as_ref()
637            .and_then(|store| store.context_artifact_store())
638            .or_else(|| {
639                thread_store
640                    .as_ref()
641                    .and_then(|store| store.local_thread_root())
642                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
643            })
644            .unwrap_or_else(|| {
645                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
646            });
647        let goals = Arc::new(RuntimeGoalController::new(
648            bus.clone(),
649            thread_store.clone(),
650        ));
651        let runtime = Self {
652            bus,
653            registry,
654            config: RwLock::new(config),
655            pending_plan_exit: RwLock::new(None),
656            pending_tool_approvals: Mutex::new(HashMap::new()),
657            pending_user_inputs: Mutex::new(HashMap::new()),
658            pending_external_tool_calls: Mutex::new(HashMap::new()),
659            turn_admission: Mutex::new(()),
660            active_turns: RwLock::new(HashMap::new()),
661            turn_drains: RwLock::new(HashMap::new()),
662            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
663            accepting_turns: AtomicBool::new(true),
664            lifecycle_persistence_failures: AtomicUsize::new(0),
665            lifecycle_shutdown_drains: AtomicUsize::new(0),
666            lifecycle_clean_shutdowns: AtomicUsize::new(0),
667            lifecycle_deadline_exceeded: AtomicUsize::new(0),
668            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
669            lifecycle_restart_reconciliations: AtomicUsize::new(0),
670            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
671            provider_cleanup_confirmed: AtomicUsize::new(0),
672            provider_cleanup_timed_out: AtomicUsize::new(0),
673            provider_cleanup_unknown: AtomicUsize::new(0),
674            active_turns_changed: Notify::new(),
675            active_turn_selections: RwLock::new(HashMap::new()),
676            active_turn_contexts: RwLock::new(HashMap::new()),
677            allow_local_workspaces: AtomicBool::new(true),
678            team_member_turn_contexts: Mutex::new(HashMap::new()),
679            workspace: workspace.clone(),
680            teams: TeamManager::new(
681                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
682            ),
683            agent_team_spawn_lock: Mutex::new(()),
684            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
685                workspace,
686                roadmap_data_dir,
687            )),
688            goals,
689            context_artifacts,
690            thread_store,
691            thread_item_cache: Mutex::new(ThreadItemCache::default()),
692            tool_registry,
693            media_generation,
694            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
695                PathBuf::new(),
696            ))),
697            event_sink_dispatcher: tokio::sync::OnceCell::new(),
698            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
699            agent_swarm_modes: RwLock::new(HashMap::new()),
700            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
701        };
702        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
703            timestamp: OffsetDateTime::now_utc(),
704        }));
705        for manifest in &runtime.registry.manifests {
706            runtime
707                .bus
708                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
709                    extension_id: manifest.id.clone(),
710                    timestamp: OffsetDateTime::now_utc(),
711                }));
712        }
713        Ok(runtime)
714    }
715
716    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
717        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
718        builder.inference_engine(engine);
719        Self::new(builder.build()?, RuntimeConfig::default())
720    }
721
722    pub fn fake() -> anyhow::Result<Self> {
723        Self::from_engine(Arc::new(FakeInferenceEngine))
724    }
725
726    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
727        self.bus.subscribe()
728    }
729
730    /// Returns process-local, redacted lifecycle health counters. The snapshot
731    /// intentionally has fixed fields and contains no provider, command,
732    /// process, thread, or turn identifiers.
733    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
734        LifecycleMetricsSnapshot {
735            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
736            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
737            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
738                as u64,
739            persistence_failed_count: self
740                .lifecycle_persistence_failed_drains
741                .load(Ordering::Acquire) as u64,
742            restart_reconciliation_count: self
743                .lifecycle_restart_reconciliations
744                .load(Ordering::Acquire) as u64,
745            lifecycle_persistence_failure_count: self
746                .lifecycle_persistence_failures
747                .load(Ordering::Acquire) as u64,
748            shutdown_drain_duration_ms_total: self
749                .lifecycle_shutdown_drain_duration_ms_total
750                .load(Ordering::Acquire) as u64,
751            provider_cleanup_confirmed_count: self
752                .provider_cleanup_confirmed
753                .load(Ordering::Acquire) as u64,
754            provider_cleanup_timed_out_count: self
755                .provider_cleanup_timed_out
756                .load(Ordering::Acquire) as u64,
757            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
758                as u64,
759        }
760    }
761
762    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
763        let Some(store) = &self.thread_store else {
764            return true;
765        };
766        let state = match record.extension_state() {
767            Ok(state) => state,
768            Err(_) => {
769                self.lifecycle_persistence_failures
770                    .fetch_add(1, Ordering::AcqRel);
771                return false;
772            }
773        };
774        // Lifecycle diagnostics must not turn a provider result into a failed
775        // turn merely because a third-party store lacks extension-state support.
776        // JSONL/Postgres stores persist this append-only record atomically at
777        // their own storage boundary.
778        if store
779            .append_extension_state(&record.thread_id, &state)
780            .await
781            .is_err()
782        {
783            self.lifecycle_persistence_failures
784                .fetch_add(1, Ordering::AcqRel);
785            return false;
786        }
787        true
788    }
789
790    async fn record_turn_lifecycle(
791        &self,
792        thread_id: ThreadId,
793        turn_id: TurnId,
794        state: TurnLifecycleState,
795        cleanup: TurnCleanupState,
796        reason: Option<TurnLifecycleReason>,
797    ) -> TurnLifecycleRecord {
798        self.record_turn_lifecycle_with_ownership(
799            thread_id,
800            turn_id,
801            state,
802            cleanup,
803            reason,
804            TurnCleanupOwnership::RuntimeTaskOnly,
805        )
806        .await
807    }
808
809    async fn record_turn_lifecycle_with_ownership(
810        &self,
811        thread_id: ThreadId,
812        turn_id: TurnId,
813        state: TurnLifecycleState,
814        cleanup: TurnCleanupState,
815        reason: Option<TurnLifecycleReason>,
816        ownership: TurnCleanupOwnership,
817    ) -> TurnLifecycleRecord {
818        let record = TurnLifecycleRecord::new(
819            thread_id,
820            turn_id,
821            state,
822            cleanup,
823            reason,
824            OffsetDateTime::now_utc(),
825        )
826        .with_ownership(ownership);
827        let _ = self.persist_turn_lifecycle_record(&record).await;
828        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
829            .await;
830        record
831    }
832
833    pub(crate) fn register_provider_turn_cleanup(
834        &self,
835        turn_id: &TurnId,
836        cleanup: Arc<dyn ProviderTurnCleanup>,
837    ) {
838        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
839            cleanups.insert(turn_id.clone(), cleanup);
840        }
841    }
842
843    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
844        self.provider_turn_cleanups
845            .lock()
846            .ok()
847            .and_then(|cleanups| cleanups.get(turn_id).cloned())
848            .map(|cleanup| cleanup.ownership())
849            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
850    }
851
852    async fn await_provider_turn_cleanup(
853        &self,
854        turn_id: &TurnId,
855    ) -> (TurnCleanupState, TurnCleanupOwnership) {
856        let cleanup = self
857            .provider_turn_cleanups
858            .lock()
859            .ok()
860            .and_then(|mut cleanups| cleanups.remove(turn_id));
861        let Some(cleanup) = cleanup else {
862            return (
863                TurnCleanupState::Unknown,
864                TurnCleanupOwnership::RuntimeTaskOnly,
865            );
866        };
867        let ownership = cleanup.ownership();
868        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
869            Ok(Ok(())) => {
870                self.provider_cleanup_confirmed
871                    .fetch_add(1, Ordering::AcqRel);
872                (
873                    TurnCleanupState::Completed,
874                    TurnCleanupOwnership::ProviderCleanupConfirmed,
875                )
876            }
877            Ok(Err(_)) => {
878                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
879                (TurnCleanupState::Unknown, ownership)
880            }
881            Err(_) => {
882                self.provider_cleanup_timed_out
883                    .fetch_add(1, Ordering::AcqRel);
884                (TurnCleanupState::TimedOut, ownership)
885            }
886        }
887    }
888
889    fn record_lifecycle_drain_outcome(
890        &self,
891        started_at: tokio::time::Instant,
892        outcome: &RuntimeDrainOutcome,
893    ) {
894        self.lifecycle_shutdown_drains
895            .fetch_add(1, Ordering::AcqRel);
896        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
897            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
898            Ordering::AcqRel,
899        );
900        match outcome {
901            RuntimeDrainOutcome::Clean { .. } => {
902                self.lifecycle_clean_shutdowns
903                    .fetch_add(1, Ordering::AcqRel);
904            }
905            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
906                self.lifecycle_deadline_exceeded
907                    .fetch_add(1, Ordering::AcqRel);
908            }
909            RuntimeDrainOutcome::PersistenceFailed { .. } => {
910                self.lifecycle_persistence_failed_drains
911                    .fetch_add(1, Ordering::AcqRel);
912            }
913        }
914    }
915
916    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
917        match event {
918            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
919                event.thread_id.clone(),
920                event.turn_id.clone(),
921                TurnLifecycleState::Running,
922                TurnCleanupState::NotRequested,
923                None,
924                event.timestamp,
925            )),
926            // `run_turn` persists completed turns after it has awaited a
927            // registered provider cleanup handle. Do not let the generic event
928            // projection overwrite that acknowledgement with an unproven state.
929            RoderEvent::TurnCompleted(_) => None,
930            // Some failure paths already have an event before the turn wrapper
931            // reaches its cleanup acknowledgement. Preserve a durable fallback
932            // reason here; the wrapper appends a later record with the more
933            // precise cleanup outcome when a provider registered one.
934            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
935                event.thread_id.clone(),
936                event.turn_id.clone(),
937                TurnLifecycleState::Failed,
938                TurnCleanupState::Unknown,
939                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
940                    TurnLifecycleReason::DeadlineExceeded
941                } else {
942                    TurnLifecycleReason::ProviderFailure
943                }),
944                event.timestamp,
945            )),
946            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
947            RoderEvent::TurnLifecycleUpdated(_) => None,
948            _ => None,
949        }
950    }
951
952    pub fn registry(&self) -> &ExtensionRegistry {
953        &self.registry
954    }
955
956    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
957        self.media_generation.clone()
958    }
959
960    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
961        self.context_artifacts.clone()
962    }
963
964    pub async fn execute_workflow_tool(
965        &self,
966        thread_id: ThreadId,
967        tool_name: &str,
968        arguments: serde_json::Value,
969    ) -> anyhow::Result<ToolResult> {
970        let Some(executor) = self.tool_registry.get(tool_name) else {
971            anyhow::bail!("tool not found: {tool_name}");
972        };
973        let tool_call = ToolCall {
974            id: format!("slash-{tool_name}"),
975            name: tool_name.to_string(),
976            raw_arguments: serde_json::to_string(&arguments)?,
977            arguments,
978            thread_id: thread_id.clone(),
979            turn_id: "slash-command".to_string(),
980        };
981        let runtime_config = self.status().await;
982        let ctx = self.tool_execution_context(
983            thread_id,
984            "slash-command".to_string(),
985            runtime_config.policy_mode,
986            runtime_config.workspace.as_deref(),
987            Some(&runtime_config.command_shell),
988        );
989        executor.execute(ctx, tool_call).await
990    }
991
992    pub(crate) fn tool_execution_context(
993        &self,
994        thread_id: ThreadId,
995        turn_id: TurnId,
996        mode: PolicyMode,
997        workspace: Option<&str>,
998        command_shell: Option<&str>,
999    ) -> ToolExecutionContext {
1000        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1001            .with_command_shell(command_shell.unwrap_or_default())
1002            .with_process_runner(Arc::new(LocalProcessRunner))
1003            .with_context_artifacts(self.context_artifacts.backend())
1004            .with_goal_controller(self.goals.clone())
1005            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1006                self.bus.clone(),
1007                self.thread_store.clone(),
1008            )))
1009            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1010                self.bus.clone(),
1011                self.thread_store.clone(),
1012            )));
1013        if let Some(workspace) = workspace {
1014            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1015        }
1016        ctx
1017    }
1018
1019    pub async fn status(&self) -> RuntimeConfig {
1020        self.config.read().await.clone()
1021    }
1022
1023    pub async fn set_skills(&self, skills: SkillRegistry) {
1024        *self.skills.write().await = skills;
1025    }
1026
1027    pub async fn skills_snapshot(&self) -> SkillRegistry {
1028        self.skills.read().await.clone()
1029    }
1030
1031    pub fn workspace(&self) -> PathBuf {
1032        self.workspace.clone()
1033    }
1034
1035    /// Controls whether native workspace tools may execute without an
1036    /// explicit remote-runner binding. Hosted runtimes disable this after
1037    /// construction; ordinary local runtimes retain the compatible default.
1038    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1039        self.allow_local_workspaces
1040            .store(allowed, Ordering::Relaxed);
1041    }
1042
1043    pub fn allows_local_workspaces(&self) -> bool {
1044        self.allow_local_workspaces.load(Ordering::Relaxed)
1045    }
1046
1047    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1048        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1049            destination_id: destination.id.clone(),
1050            provider_id: destination.provider_id.clone(),
1051            state: "configured".to_string(),
1052            session_id: None,
1053            timestamp: OffsetDateTime::now_utc(),
1054        });
1055        self.config.write().await.remote_runner_destination = destination;
1056        if let Some(lifecycle) = lifecycle {
1057            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1058        } else {
1059            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1060                destination_id: "local".to_string(),
1061                provider_id: "local".to_string(),
1062                state: "local_fallback".to_string(),
1063                session_id: None,
1064                timestamp: OffsetDateTime::now_utc(),
1065            }))
1066            .await;
1067        }
1068    }
1069
1070    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1071        let mut cfg = self.config.write().await;
1072        cfg.file_backed_dynamic_context = enabled;
1073        cfg.clone()
1074    }
1075
1076    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1077        let mut cfg = self.config.write().await;
1078        cfg.command_shell = shell;
1079        cfg.clone()
1080    }
1081
1082    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1083        let mut pending = self.pending_plan_exit.write().await;
1084        let current = pending.clone()?;
1085        if !current.is_expired(OffsetDateTime::now_utc()) {
1086            return Some(current);
1087        }
1088        *pending = None;
1089        drop(pending);
1090        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1091            .await;
1092        None
1093    }
1094
1095    pub async fn set_policy_mode(
1096        &self,
1097        mode: PolicyMode,
1098        reason: Option<String>,
1099    ) -> anyhow::Result<RuntimeConfig> {
1100        let mut cfg = self.config.write().await;
1101        let previous_mode = cfg.policy_mode;
1102        cfg.policy_mode = mode;
1103        let next = cfg.clone();
1104        drop(cfg);
1105        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1106            thread_id: "runtime".to_string(),
1107            turn_id: None,
1108            previous_mode,
1109            new_mode: mode,
1110            reason,
1111            timestamp: OffsetDateTime::now_utc(),
1112        }))
1113        .await;
1114        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1115            .await;
1116        Ok(next)
1117    }
1118
1119    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1120    /// swarm reminder is injected into each turn's developer instructions so
1121    /// every app-server/SDK client gets it, not only the TUI.
1122    pub async fn set_agent_swarm_mode(
1123        &self,
1124        enabled: bool,
1125        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1126    ) -> anyhow::Result<RuntimeConfig> {
1127        let mut cfg = self.config.write().await;
1128        cfg.agent_swarm_mode = enabled;
1129        let next = cfg.clone();
1130        drop(cfg);
1131        self.emit(RoderEvent::AgentSwarmModeChanged(
1132            roder_api::subagents::AgentSwarmModeChanged {
1133                thread_id: "runtime".to_string(),
1134                turn_id: None,
1135                enabled,
1136                trigger,
1137                timestamp: OffsetDateTime::now_utc(),
1138            },
1139        ))
1140        .await;
1141        Ok(next)
1142    }
1143
1144    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1145    /// stored per thread so toggling swarm mode on one thread does not leak the
1146    /// reminder into other threads sharing the runtime; absent threads fall back
1147    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1148    /// carries the real `thread_id`.
1149    pub async fn set_agent_swarm_mode_for_thread(
1150        &self,
1151        thread_id: &str,
1152        enabled: bool,
1153        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1154    ) -> bool {
1155        {
1156            let mut modes = self.agent_swarm_modes.write().await;
1157            modes.insert(thread_id.to_string(), enabled);
1158        }
1159        self.emit(RoderEvent::AgentSwarmModeChanged(
1160            roder_api::subagents::AgentSwarmModeChanged {
1161                thread_id: thread_id.to_string(),
1162                turn_id: None,
1163                enabled,
1164                trigger,
1165                timestamp: OffsetDateTime::now_utc(),
1166            },
1167        ))
1168        .await;
1169        enabled
1170    }
1171
1172    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1173    /// override when present, otherwise the runtime-global default. The turn loop
1174    /// uses this to decide whether to inject the swarm reminder.
1175    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1176        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1177            return enabled;
1178        }
1179        self.status().await.agent_swarm_mode
1180    }
1181
1182    pub async fn set_hosted_web_search(
1183        &self,
1184        mode: HostedWebSearchMode,
1185    ) -> anyhow::Result<RuntimeConfig> {
1186        let mut cfg = self.config.write().await;
1187        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1188        Ok(cfg.clone())
1189    }
1190
1191    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1192        let gate = DefaultPolicyGate::new();
1193        let mut pending = self.pending_tool_approvals.lock().await;
1194        let approval_ids = pending
1195            .iter()
1196            .filter_map(|(approval_id, approval)| {
1197                let ctx = ToolExecutionContext::new(
1198                    approval.thread_id.clone(),
1199                    approval.turn_id.clone(),
1200                    mode,
1201                );
1202                matches!(
1203                    gate.decide(&approval.call, mode, &ctx),
1204                    PolicyDecision::AutoApproved { .. }
1205                )
1206                .then_some(approval_id.clone())
1207            })
1208            .collect::<Vec<_>>();
1209        let approvals = approval_ids
1210            .into_iter()
1211            .filter_map(|approval_id| {
1212                pending
1213                    .remove(&approval_id)
1214                    .map(|approval| (approval_id, approval))
1215            })
1216            .collect::<Vec<_>>();
1217        drop(pending);
1218
1219        for (approval_id, approval) in approvals {
1220            let ctx = ToolExecutionContext::new(
1221                approval.thread_id.clone(),
1222                approval.turn_id.clone(),
1223                mode,
1224            );
1225            let decision = gate.decide(&approval.call, mode, &ctx);
1226            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1227                thread_id: approval.thread_id.clone(),
1228                turn_id: approval.turn_id.clone(),
1229                tool_id: approval.tool_id.clone(),
1230                tool_name: approval.tool_name.clone(),
1231                mode,
1232                decision,
1233                timestamp: OffsetDateTime::now_utc(),
1234            }))
1235            .await;
1236            if mode == PolicyMode::Bypass {
1237                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1238                    thread_id: approval.thread_id.clone(),
1239                    turn_id: approval.turn_id.clone(),
1240                    tool_id: approval.tool_id.clone(),
1241                    tool_name: approval.tool_name.clone(),
1242                    timestamp: OffsetDateTime::now_utc(),
1243                }))
1244                .await;
1245            }
1246            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1247                thread_id: approval.thread_id,
1248                turn_id: approval.turn_id,
1249                approval_id,
1250                tool_id: approval.tool_id,
1251                tool_name: approval.tool_name,
1252                approved: true,
1253                timestamp: OffsetDateTime::now_utc(),
1254            }))
1255            .await;
1256            let _ = approval.tx.send(true);
1257        }
1258    }
1259
1260    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1261        *self.pending_plan_exit.write().await = Some(pending.clone());
1262        self.emit(RoderEvent::PolicyExitPlanRequested(
1263            PolicyExitPlanRequested {
1264                thread_id: pending.thread_id,
1265                turn_id: pending.turn_id,
1266                request_id: pending.request_id,
1267                target_mode: pending.target_mode,
1268                plan_summary: pending.plan_summary,
1269                next_steps: pending.next_steps,
1270                timestamp: OffsetDateTime::now_utc(),
1271            },
1272        ))
1273        .await;
1274    }
1275
1276    pub async fn resolve_pending_plan_exit(
1277        &self,
1278        request_id: &str,
1279        approved: bool,
1280    ) -> anyhow::Result<Option<PendingPlanExit>> {
1281        let mut pending = self.pending_plan_exit.write().await;
1282        let Some(current) = pending.clone() else {
1283            return Ok(None);
1284        };
1285        if current.request_id != request_id {
1286            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1287        }
1288        *pending = None;
1289        drop(pending);
1290
1291        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1292        let resolved_mode = if approved {
1293            let mut cfg = self.config.write().await;
1294            let previous_mode = cfg.policy_mode;
1295            cfg.policy_mode = current.target_mode;
1296            drop(cfg);
1297            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1298                thread_id: current.thread_id.clone(),
1299                turn_id: Some(current.turn_id.clone()),
1300                previous_mode,
1301                new_mode: current.target_mode,
1302                reason: Some("approved plan exit".to_string()),
1303                timestamp: OffsetDateTime::now_utc(),
1304            }))
1305            .await;
1306            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1307                .await;
1308            current.target_mode
1309        } else {
1310            self.status().await.policy_mode
1311        };
1312        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1313            .await;
1314        Ok(Some(current))
1315    }
1316
1317    pub async fn resolve_tool_approval(
1318        &self,
1319        approval_id: &str,
1320        approved: bool,
1321    ) -> anyhow::Result<bool> {
1322        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1323        let Some(pending) = pending else {
1324            return Ok(false);
1325        };
1326        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1327            thread_id: pending.thread_id,
1328            turn_id: pending.turn_id,
1329            approval_id: approval_id.to_string(),
1330            tool_id: pending.tool_id,
1331            tool_name: pending.tool_name,
1332            approved,
1333            timestamp: OffsetDateTime::now_utc(),
1334        }))
1335        .await;
1336        let _ = pending.tx.send(approved);
1337        Ok(true)
1338    }
1339
1340    pub async fn request_app_server_tool_approval(
1341        &self,
1342        call: ToolCall,
1343        reason: Option<String>,
1344    ) -> anyhow::Result<bool> {
1345        let approval_id = call.id.clone();
1346        let (tx, rx) = oneshot::channel();
1347        self.pending_tool_approvals.lock().await.insert(
1348            approval_id.clone(),
1349            PendingToolApproval {
1350                thread_id: call.thread_id.clone(),
1351                turn_id: call.turn_id.clone(),
1352                tool_id: call.id.clone(),
1353                tool_name: call.name.clone(),
1354                call: call.clone(),
1355                tx,
1356            },
1357        );
1358        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1359            thread_id: call.thread_id.clone(),
1360            turn_id: call.turn_id.clone(),
1361            approval_id,
1362            tool_id: call.id.clone(),
1363            tool_name: call.name.clone(),
1364            reason,
1365            timestamp: OffsetDateTime::now_utc(),
1366        }))
1367        .await;
1368        Ok(rx.await.unwrap_or(false))
1369    }
1370
1371    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1372    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1373    pub async fn resolve_external_tool_call(
1374        &self,
1375        request_id: &str,
1376        resolution: ExternalToolResolution,
1377    ) -> anyhow::Result<bool> {
1378        let pending = self
1379            .pending_external_tool_calls
1380            .lock()
1381            .await
1382            .remove(request_id);
1383        let Some(pending) = pending else {
1384            return Ok(false);
1385        };
1386        self.emit(RoderEvent::ExternalToolCallResolved(
1387            ExternalToolCallResolved {
1388                thread_id: pending.thread_id,
1389                turn_id: pending.turn_id,
1390                request_id: request_id.to_string(),
1391                tool_id: pending.tool_id,
1392                tool_name: pending.tool_name,
1393                outcome: ExternalToolCallOutcome::Resolved,
1394                is_error: resolution.is_error,
1395                timestamp: OffsetDateTime::now_utc(),
1396            },
1397        ))
1398        .await;
1399        let _ = pending.tx.send(resolution);
1400        Ok(true)
1401    }
1402
1403    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1404    /// interrupt does not leak requests waiting on `tools/resolve`.
1405    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1406        let cancelled = {
1407            let mut pending = self.pending_external_tool_calls.lock().await;
1408            let request_ids = pending
1409                .iter()
1410                .filter(|(_, call)| &call.turn_id == turn_id)
1411                .map(|(request_id, _)| request_id.clone())
1412                .collect::<Vec<_>>();
1413            request_ids
1414                .into_iter()
1415                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1416                .collect::<Vec<_>>()
1417        };
1418        for (request_id, call) in cancelled {
1419            self.emit(RoderEvent::ExternalToolCallResolved(
1420                ExternalToolCallResolved {
1421                    thread_id: call.thread_id,
1422                    turn_id: call.turn_id,
1423                    request_id,
1424                    tool_id: call.tool_id,
1425                    tool_name: call.tool_name,
1426                    outcome: ExternalToolCallOutcome::Cancelled,
1427                    is_error: true,
1428                    timestamp: OffsetDateTime::now_utc(),
1429                },
1430            ))
1431            .await;
1432        }
1433    }
1434
1435    pub async fn resolve_user_input(
1436        &self,
1437        request_id: &str,
1438        answers: serde_json::Value,
1439    ) -> anyhow::Result<bool> {
1440        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1441        let Some(pending) = pending else {
1442            return Ok(false);
1443        };
1444        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1445            thread_id: pending.thread_id,
1446            turn_id: pending.turn_id,
1447            request_id: request_id.to_string(),
1448            answers: answers.clone(),
1449            timestamp: OffsetDateTime::now_utc(),
1450        }))
1451        .await;
1452        let _ = pending.tx.send(answers);
1453        Ok(true)
1454    }
1455
1456    async fn emit_plan_exit_resolved(
1457        &self,
1458        current: &PendingPlanExit,
1459        approved: bool,
1460        resolved_mode: PolicyMode,
1461    ) {
1462        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1463            thread_id: current.thread_id.clone(),
1464            turn_id: current.turn_id.clone(),
1465            request_id: current.request_id.clone(),
1466            approved,
1467            target_mode: current.target_mode,
1468            resolved_mode,
1469            timestamp: OffsetDateTime::now_utc(),
1470        }))
1471        .await;
1472    }
1473
1474    pub async fn select_provider(
1475        &self,
1476        provider: String,
1477        model: Option<String>,
1478        reasoning: Option<String>,
1479    ) -> anyhow::Result<RuntimeConfig> {
1480        let next = self
1481            .preview_provider_selection(provider, model, reasoning)
1482            .await?;
1483        let mut cfg = self.config.write().await;
1484        *cfg = next;
1485        Ok(cfg.clone())
1486    }
1487
1488    pub async fn preview_provider_selection(
1489        &self,
1490        provider: String,
1491        model: Option<String>,
1492        reasoning: Option<String>,
1493    ) -> anyhow::Result<RuntimeConfig> {
1494        self.engine_for(&provider)?;
1495        let mut cfg = self.config.read().await.clone();
1496        cfg.default_provider = provider;
1497        if let Some(model) = model {
1498            cfg.default_model = model;
1499        }
1500        if let Some(reasoning) = reasoning {
1501            if reasoning == REASONING_NONE
1502                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1503            {
1504                return Ok(cfg.clone());
1505            }
1506            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1507            cfg.reasoning = Some(reasoning);
1508        }
1509        Ok(cfg)
1510    }
1511
1512    pub async fn effective_reasoning(&self) -> String {
1513        let cfg = self.config.read().await;
1514        effective_reasoning_for_model(&cfg, &cfg.default_model)
1515    }
1516
1517    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1518        effective_reasoning_for_model(cfg, &cfg.default_model)
1519    }
1520
1521    pub async fn set_dynamic_workflow_effort(
1522        &self,
1523        effort_profile: DynamicWorkflowEffortProfile,
1524    ) -> RuntimeConfig {
1525        let mut cfg = self.config.write().await;
1526        cfg.dynamic_workflows.effort_profile = effort_profile;
1527        cfg.clone()
1528    }
1529
1530    pub async fn dynamic_workflow_trigger_decision(
1531        &self,
1532        message: &str,
1533    ) -> WorkflowTriggerDecision {
1534        let cfg = self.config.read().await;
1535        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1536    }
1537
1538    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1539        self.create_thread_with(CreateThreadRequest {
1540            title,
1541            workspace: self.workspace.display().to_string(),
1542            workspace_id: None,
1543            root_id: None,
1544            provider: None,
1545            model: None,
1546            selection_mode: None,
1547            tool_allowlist: Vec::new(),
1548            developer_instructions: None,
1549            external_tools: Vec::new(),
1550            runner: None,
1551        })
1552        .await
1553    }
1554
1555    /**
1556     * Resolves an explicit thread-runner selection into a persisted binding.
1557     * Fails fast at thread creation when the provider is missing or rejects
1558     * the destination, instead of surfacing the error on the first tool call.
1559     */
1560    async fn resolve_thread_runner_binding(
1561        &self,
1562        thread_id: &str,
1563        selection: ThreadRunnerSelection,
1564    ) -> anyhow::Result<ThreadRunnerBinding> {
1565        let provider = self
1566            .registry
1567            .remote_runner_providers
1568            .iter()
1569            .find(|provider| provider.id() == selection.provider_id)
1570            .cloned()
1571            .ok_or_else(|| {
1572                anyhow::anyhow!(
1573                    "remote runner provider {:?} is not installed",
1574                    selection.provider_id
1575                )
1576            })?;
1577        let workspace = selection.workspace.trim();
1578        anyhow::ensure!(
1579            std::path::Path::new(workspace).is_absolute(),
1580            "runner workspace must be an absolute path on the runner: {workspace:?}"
1581        );
1582        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1583        for read_root in &selection.read_roots {
1584            let trimmed = read_root.trim();
1585            anyhow::ensure!(
1586                std::path::Path::new(trimmed).is_absolute(),
1587                "runner read root must be an absolute path on the runner: {trimmed:?}"
1588            );
1589            read_roots.push(PathBuf::from(trimmed));
1590        }
1591        let destination = RunnerDestination {
1592            id: format!("thread-{thread_id}"),
1593            provider_id: selection.provider_id,
1594            config: selection.config,
1595            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1596        };
1597        provider.validate_destination(&destination).await?;
1598        Ok(ThreadRunnerBinding {
1599            destination,
1600            workspace: PathBuf::from(workspace),
1601            read_roots,
1602        })
1603    }
1604
1605    /**
1606     * Build a per-thread binding from a runtime-level destination (selected via
1607     * the TUI runner picker or config `default_destination`) when the
1608     * destination's provider advertises a default workspace. Returns `None` when
1609     * there is no destination or the provider opts out (no default workspace),
1610     * preserving the legacy behavior where such threads keep local tools.
1611     */
1612    async fn synthesize_runtime_runner_binding(
1613        &self,
1614        thread_id: &str,
1615        destination: Option<RunnerDestination>,
1616    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1617        let Some(destination) = destination else {
1618            return Ok(None);
1619        };
1620        let Some(provider) = self
1621            .registry
1622            .remote_runner_providers
1623            .iter()
1624            .find(|provider| provider.id() == destination.provider_id)
1625        else {
1626            return Ok(None);
1627        };
1628        // An explicit workspace in the destination config wins over the
1629        // provider default; absence of both means the provider opts out.
1630        let workspace = destination
1631            .config
1632            .get("working_dir")
1633            .and_then(serde_json::Value::as_str)
1634            .map(str::to_string)
1635            .or_else(|| provider.default_workspace());
1636        let Some(workspace) = workspace else {
1637            return Ok(None);
1638        };
1639        let selection = ThreadRunnerSelection {
1640            provider_id: destination.provider_id.clone(),
1641            config: destination.config.clone(),
1642            workspace,
1643            read_roots: Vec::new(),
1644        };
1645        Ok(Some(
1646            self.resolve_thread_runner_binding(thread_id, selection)
1647                .await?,
1648        ))
1649    }
1650
1651    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1652    pub async fn validate_thread_runner_selection(
1653        &self,
1654        selection: ThreadRunnerSelection,
1655    ) -> anyhow::Result<()> {
1656        self.resolve_thread_runner_binding("validate", selection)
1657            .await
1658            .map(|_| ())
1659    }
1660
1661    pub async fn create_thread_with(
1662        &self,
1663        req: CreateThreadRequest,
1664    ) -> anyhow::Result<ThreadMetadata> {
1665        let cfg = self.config.read().await.clone();
1666        let now = OffsetDateTime::now_utc();
1667        let workspace = validate_thread_workspace(&req.workspace)?;
1668        let provider = req.provider.unwrap_or(cfg.default_provider);
1669        let model = req.model.unwrap_or(cfg.default_model);
1670        let selection_mode = req
1671            .selection_mode
1672            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1673        let thread_id = uuid::Uuid::new_v4().to_string();
1674        let runner_binding = match req.runner {
1675            Some(selection) => Some(
1676                self.resolve_thread_runner_binding(&thread_id, selection)
1677                    .await?,
1678            ),
1679            // No explicit per-thread selection: if a runtime-level destination
1680            // is active and its provider advertises a default workspace, bind
1681            // the new thread so its coding tools route into the runner. This is
1682            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1683            // execute tools in the sandbox instead of locally. Providers that
1684            // return no default workspace stay unbound (legacy local-tools).
1685            None => {
1686                self.synthesize_runtime_runner_binding(
1687                    &thread_id,
1688                    cfg.remote_runner_destination.clone(),
1689                )
1690                .await?
1691            }
1692        };
1693        let runner_destination = runner_binding
1694            .as_ref()
1695            .map(|binding| binding.destination.clone())
1696            .or_else(|| cfg.remote_runner_destination.clone());
1697        let metadata = ThreadMetadata {
1698            thread_id,
1699            title: req.title,
1700            workspace,
1701            workspace_id: req.workspace_id,
1702            root_id: req.root_id,
1703            provider: Some(provider),
1704            model: Some(model),
1705            selection_mode: Some(selection_mode),
1706            tool_allowlist: req.tool_allowlist,
1707            developer_instructions: req.developer_instructions,
1708            external_tools: req.external_tools,
1709            runner_destination,
1710            runner_state: None,
1711            runner_binding,
1712            created_at: now,
1713            updated_at: now,
1714            message_count: 0,
1715            usage: None,
1716            parent_thread_id: None,
1717            forked_from_turn_id: None,
1718            workspace_fork: None,
1719        };
1720
1721        let metadata = if let Some(store) = &self.thread_store {
1722            store.create_thread(metadata).await?
1723        } else {
1724            metadata
1725        };
1726        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1727            thread_id: metadata.thread_id.clone(),
1728            timestamp: OffsetDateTime::now_utc(),
1729        }))
1730        .await;
1731        Ok(metadata)
1732    }
1733
1734    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1735        if let Some(store) = &self.thread_store {
1736            return store.list_threads().await;
1737        }
1738        Ok(Vec::new())
1739    }
1740
1741    pub async fn list_threads_page(
1742        &self,
1743        options: roder_api::thread::ThreadListOptions,
1744    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1745        if let Some(store) = &self.thread_store {
1746            return store.list_threads_page(options).await;
1747        }
1748        Ok(roder_api::thread::ThreadListPage::default())
1749    }
1750
1751    pub async fn load_thread_metadata(
1752        &self,
1753        thread_id: &str,
1754    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1755        if let Some(store) = &self.thread_store {
1756            return store.load_thread_metadata(&thread_id.to_string()).await;
1757        }
1758        Ok(None)
1759    }
1760
1761    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1762        let archived = if let Some(store) = &self.thread_store {
1763            store.archive_thread(&thread_id.to_string()).await?
1764        } else {
1765            false
1766        };
1767        if archived {
1768            self.thread_item_cache
1769                .lock()
1770                .await
1771                .remove_thread(&thread_id.to_string());
1772            self.evict_runner_session(&thread_id.to_string()).await;
1773        }
1774        Ok(archived)
1775    }
1776
1777    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1778        let cfg = self.config.read().await.clone();
1779        let workspace = self.workspace.display().to_string();
1780        let lead_thread_id = match req.lead_thread_id {
1781            Some(thread_id) => thread_id,
1782            None => {
1783                self.create_thread_with(CreateThreadRequest {
1784                    title: Some("Team lead".to_string()),
1785                    workspace: workspace.clone(),
1786                    workspace_id: None,
1787                    root_id: None,
1788                    provider: None,
1789                    model: None,
1790                    selection_mode: None,
1791                    tool_allowlist: Vec::new(),
1792                    developer_instructions: None,
1793                    external_tools: Vec::new(),
1794                    runner: None,
1795                })
1796                .await?
1797                .thread_id
1798            }
1799        };
1800        let team_id = uuid::Uuid::new_v4().to_string();
1801        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1802        let lead_selection = match active_lead_turn_id.as_ref() {
1803            Some(turn_id) => self
1804                .active_turn_selections
1805                .read()
1806                .await
1807                .get(turn_id)
1808                .cloned(),
1809            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1810        };
1811        let lead_concrete_selection = lead_selection
1812            .as_ref()
1813            .map(ModelSelectionMode::concrete_selection);
1814        let mut lead = crate::teams::lead_member(
1815            lead_thread_id.clone(),
1816            lead_concrete_selection
1817                .as_ref()
1818                .map(|selection| selection.provider.clone())
1819                .or_else(|| Some(cfg.default_provider.clone())),
1820            lead_concrete_selection
1821                .as_ref()
1822                .map(|selection| selection.model.clone())
1823                .or_else(|| Some(cfg.default_model.clone())),
1824            cfg.policy_mode,
1825        );
1826        if let Some(turn_id) = active_lead_turn_id {
1827            lead.current_turn_id = Some(turn_id);
1828            lead.status = TeamMemberStatus::Running;
1829        }
1830        let mut members = vec![lead];
1831
1832        for (index, member) in req.members.into_iter().enumerate() {
1833            let thread = self
1834                .create_thread_with(CreateThreadRequest {
1835                    title: Some(member.name.clone()),
1836                    workspace: workspace.clone(),
1837                    workspace_id: None,
1838                    root_id: None,
1839                    provider: member.model_provider.clone(),
1840                    model: member.model.clone(),
1841                    selection_mode: None,
1842                    tool_allowlist: Vec::new(),
1843                    developer_instructions: None,
1844                    external_tools: Vec::new(),
1845                    runner: None,
1846                })
1847                .await?;
1848            let member_id = format!("member-{}", index + 1);
1849            let descriptor = crate::teams::teammate_member(
1850                member_id.clone(),
1851                member.name,
1852                thread.thread_id.clone(),
1853                member.model_provider.or(thread.provider),
1854                member.model.or(thread.model),
1855                cfg.policy_mode,
1856            );
1857            members.push(descriptor);
1858        }
1859
1860        let now = OffsetDateTime::now_utc();
1861        let team = self
1862            .teams
1863            .insert(TeamState {
1864                id: team_id.clone(),
1865                lead_thread_id: lead_thread_id.clone(),
1866                display_mode: req.display_mode,
1867                members,
1868                mailbox: Vec::new(),
1869                tasks: Vec::new(),
1870                created_at: now,
1871                updated_at: now,
1872            })
1873            .await?;
1874        self.emit(RoderEvent::TeamStarted(TeamStarted {
1875            team_id: team_id.clone(),
1876            lead_thread_id,
1877            display_mode: team.display_mode,
1878            members: team.members.clone(),
1879            tasks: team.tasks.clone(),
1880            timestamp: OffsetDateTime::now_utc(),
1881        }))
1882        .await;
1883        for member in team
1884            .members
1885            .iter()
1886            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1887        {
1888            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1889                team_id: team_id.clone(),
1890                member: member.clone(),
1891                timestamp: OffsetDateTime::now_utc(),
1892            }))
1893            .await;
1894        }
1895        Ok(team)
1896    }
1897
1898    pub async fn list_teams(&self) -> Vec<TeamState> {
1899        let active_thread_ids = self
1900            .active_turns
1901            .read()
1902            .await
1903            .values()
1904            .map(|handle| handle.thread_id.clone())
1905            .collect::<std::collections::HashSet<_>>();
1906        self.teams
1907            .list()
1908            .await
1909            .into_iter()
1910            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1911            .collect()
1912    }
1913
1914    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1915        let active_thread_ids = self
1916            .active_turns
1917            .read()
1918            .await
1919            .values()
1920            .map(|handle| handle.thread_id.clone())
1921            .collect::<std::collections::HashSet<_>>();
1922        self.teams
1923            .get(team_id)
1924            .await
1925            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1926    }
1927
1928    pub async fn start_team_member(
1929        &self,
1930        team_id: &str,
1931        req: TeamMemberStartRequest,
1932    ) -> anyhow::Result<TeamState> {
1933        self.start_team_member_with_selection(team_id, req, None)
1934            .await
1935    }
1936
1937    pub async fn message_team_member(
1938        self: &Arc<Self>,
1939        team_id: &str,
1940        member_id: &str,
1941        message: String,
1942    ) -> anyhow::Result<TurnId> {
1943        let team = self
1944            .read_team(team_id)
1945            .await
1946            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1947        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
1948            .await
1949    }
1950
1951    /// Queue a mailbox message without starting an idle agent. If the target is
1952    /// already running, the message is delivered at the next inference boundary.
1953    pub(crate) async fn queue_team_member_message(
1954        self: &Arc<Self>,
1955        caller_thread_id: &ThreadId,
1956        team_id: &str,
1957        member_id: &str,
1958        message: String,
1959    ) -> anyhow::Result<Option<TurnId>> {
1960        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
1961        let team = self
1962            .read_team(team_id)
1963            .await
1964            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1965        let member = team
1966            .members
1967            .iter()
1968            .find(|member| member.id == member_id)
1969            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
1970            .clone();
1971        if member.status == TeamMemberStatus::Closed {
1972            anyhow::bail!("subagent {} is closed", member.name);
1973        }
1974        let from_member_id = team
1975            .members
1976            .iter()
1977            .find(|candidate| candidate.thread_id == *caller_thread_id)
1978            .map(|candidate| candidate.id.clone());
1979        self.teams
1980            .append_mailbox_message(
1981                team_id,
1982                from_member_id,
1983                member_id.to_string(),
1984                TeamMailboxMessageKind::Message,
1985                message,
1986            )
1987            .await?;
1988        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
1989            return Ok(None);
1990        };
1991        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
1992            .await?;
1993        Ok(Some(turn_id))
1994    }
1995
1996    /// Deliver queued messages and start an idle agent, or steer its active turn.
1997    pub(crate) async fn followup_team_member(
1998        self: &Arc<Self>,
1999        caller_thread_id: &ThreadId,
2000        team_id: &str,
2001        member_id: &str,
2002        message: String,
2003    ) -> anyhow::Result<TurnId> {
2004        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2005        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2006            .await
2007    }
2008
2009    async fn followup_team_member_locked(
2010        self: &Arc<Self>,
2011        caller_thread_id: &ThreadId,
2012        team_id: &str,
2013        member_id: &str,
2014        message: String,
2015    ) -> anyhow::Result<TurnId> {
2016        let team = self
2017            .read_team(team_id)
2018            .await
2019            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2020        let member = team
2021            .members
2022            .iter()
2023            .find(|member| member.id == member_id)
2024            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2025            .clone();
2026        if member.status == TeamMemberStatus::Closed {
2027            anyhow::bail!("subagent {} is closed", member.name);
2028        }
2029        let from_member_id = team
2030            .members
2031            .iter()
2032            .find(|candidate| candidate.thread_id == *caller_thread_id)
2033            .map(|candidate| candidate.id.clone());
2034        self.teams
2035            .append_mailbox_message(
2036                team_id,
2037                from_member_id,
2038                member_id.to_string(),
2039                TeamMailboxMessageKind::NewTask,
2040                message,
2041            )
2042            .await?;
2043        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2044            self.deliver_pending_team_mailbox(
2045                team_id,
2046                member_id,
2047                member.thread_id,
2048                turn_id.clone(),
2049            )
2050            .await?;
2051            return Ok(turn_id);
2052        }
2053
2054        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2055            .await?;
2056        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2057        let inherited_context = self
2058            .team_member_turn_contexts
2059            .lock()
2060            .await
2061            .get(&member.thread_id)
2062            .cloned();
2063        let workspace = inherited_context
2064            .as_ref()
2065            .map(|context| context.workspace.clone())
2066            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2067            .unwrap_or_else(|| self.workspace.display().to_string());
2068        let member_thread_id = member.thread_id;
2069        let turn_id = self
2070            .start_turn(StartTurnRequest {
2071                thread_id: member_thread_id.clone(),
2072                message: String::new(),
2073                images: Vec::new(),
2074                provider_override: member.model_provider,
2075                model_override: member.model,
2076                reasoning_override: metadata
2077                    .as_ref()
2078                    .and_then(|metadata| metadata.selection_mode.as_ref())
2079                    .and_then(ModelSelectionMode::reasoning)
2080                    .map(str::to_string),
2081                workspace,
2082                instructions: inherited_context
2083                    .as_ref()
2084                    .map(|context| context.instructions.clone())
2085                    .unwrap_or_else(crate::default_instructions),
2086                developer_context: inherited_context
2087                    .as_ref()
2088                    .and_then(|context| context.developer_context.clone()),
2089                task_ledger_required: false,
2090            })
2091            .await?;
2092        Ok(turn_id)
2093    }
2094
2095    pub async fn set_team_member_policy_mode(
2096        &self,
2097        team_id: &str,
2098        member_id: &str,
2099        policy_mode: PolicyMode,
2100    ) -> anyhow::Result<TeamState> {
2101        self.teams
2102            .set_member_policy_mode(team_id, member_id, policy_mode)
2103            .await
2104    }
2105
2106    pub async fn interrupt_team_member(
2107        &self,
2108        team_id: &str,
2109        member_id: &str,
2110    ) -> anyhow::Result<Option<TurnId>> {
2111        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2112        let team = self
2113            .read_team(team_id)
2114            .await
2115            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2116        let member = team
2117            .members
2118            .iter()
2119            .find(|member| member.id == member_id)
2120            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2121            .clone();
2122        if member.status != TeamMemberStatus::Running {
2123            return Ok(None);
2124        }
2125        let Some(turn_id) = member.current_turn_id.clone() else {
2126            return Ok(None);
2127        };
2128        if self
2129            .active_turn_for_thread(&member.thread_id)
2130            .await
2131            .as_ref()
2132            != Some(&turn_id)
2133        {
2134            return Ok(None);
2135        }
2136        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2137            .await?;
2138        // Make queued mailbox work immediately eligible for the replacement turn while the
2139        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2140        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2141        // reservation acquired by its replacement.
2142        self.teams
2143            .release_mailbox_reservations_for_turn(&turn_id)
2144            .await;
2145        self.teams
2146            .update_member(team_id, member_id, |member| {
2147                member.status = TeamMemberStatus::Interrupted;
2148                member.current_turn_id = None;
2149                member.terminal_error = None;
2150            })
2151            .await?;
2152        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2153            team_id: team_id.to_string(),
2154            member_id: member_id.to_string(),
2155            member_thread_id: member.thread_id,
2156            turn_id: Some(turn_id.clone()),
2157            status: TeamMemberStatus::Interrupted,
2158            final_message: member.final_message,
2159            error: None,
2160            timestamp: OffsetDateTime::now_utc(),
2161        }))
2162        .await;
2163        Ok(Some(turn_id))
2164    }
2165
2166    pub async fn close_team_member(
2167        &self,
2168        team_id: &str,
2169        member_id: &str,
2170    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2171        let team = self
2172            .read_team(team_id)
2173            .await
2174            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2175        let member = team
2176            .members
2177            .iter()
2178            .find(|member| member.id == member_id)
2179            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2180            .clone();
2181        if member.role == roder_api::teams::TeamMemberRole::Lead {
2182            anyhow::bail!("team lead cannot be closed as a subagent");
2183        }
2184        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2185            if let Some(turn_id) = member.current_turn_id.clone() {
2186                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2187                    .await?;
2188                Some(turn_id)
2189            } else {
2190                None
2191            }
2192        } else {
2193            member.current_turn_id.clone()
2194        };
2195        let updated = self
2196            .teams
2197            .update_member(team_id, member_id, |member| {
2198                member.status = TeamMemberStatus::Closed;
2199                member.current_turn_id = None;
2200            })
2201            .await?;
2202        let closed = updated
2203            .members
2204            .iter()
2205            .find(|member| member.id == member_id)
2206            .cloned()
2207            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2208        self.team_member_turn_contexts
2209            .lock()
2210            .await
2211            .remove(&closed.thread_id);
2212        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2213            team_id: team_id.to_string(),
2214            member_id: closed.id.clone(),
2215            member_thread_id: closed.thread_id.clone(),
2216            turn_id: interrupted_turn_id,
2217            status: TeamMemberStatus::Closed,
2218            final_message: closed.final_message.clone(),
2219            error: closed.terminal_error.clone(),
2220            timestamp: OffsetDateTime::now_utc(),
2221        }))
2222        .await;
2223        Ok(closed)
2224    }
2225
2226    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2227        let Some(team) = self.read_team(team_id).await else {
2228            return Ok(false);
2229        };
2230        if !force
2231            && team
2232                .members
2233                .iter()
2234                .any(|member| member.status == TeamMemberStatus::Running)
2235        {
2236            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2237        }
2238        if force {
2239            for member in team.members.iter().filter(|member| {
2240                member.role != roder_api::teams::TeamMemberRole::Lead
2241                    && member.status == TeamMemberStatus::Running
2242            }) {
2243                if let Some(turn_id) = member
2244                    .current_turn_id
2245                    .clone()
2246                    .or(self.active_turn_for_thread(&member.thread_id).await)
2247                {
2248                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2249                }
2250            }
2251        }
2252        let removed = self.teams.remove(team_id).await?.is_some();
2253        if removed {
2254            let member_thread_ids = team
2255                .members
2256                .iter()
2257                .map(|member| member.thread_id.clone())
2258                .collect::<std::collections::HashSet<_>>();
2259            self.team_member_turn_contexts
2260                .lock()
2261                .await
2262                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2263            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2264                team_id: team_id.to_string(),
2265                forced: force,
2266                timestamp: OffsetDateTime::now_utc(),
2267            }))
2268            .await;
2269        }
2270        Ok(removed)
2271    }
2272
2273    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2274        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2275            return mode;
2276        }
2277        self.status().await.policy_mode
2278    }
2279
2280    async fn complete_team_member_turn_with_result(
2281        &self,
2282        thread_id: &ThreadId,
2283        turn_id: &TurnId,
2284        status: TeamMemberStatus,
2285        final_message: Option<String>,
2286        terminal_error: Option<String>,
2287    ) -> anyhow::Result<()> {
2288        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2289        let Some((team_id, member)) = self
2290            .teams
2291            .complete_member_turn(
2292                thread_id,
2293                turn_id,
2294                status,
2295                final_message.clone(),
2296                terminal_error.clone(),
2297            )
2298            .await?
2299        else {
2300            return Ok(());
2301        };
2302        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2303            && let Some(team) = self.read_team(&team_id).await
2304            && let Some(parent) = team
2305                .members
2306                .iter()
2307                .find(|candidate| candidate.thread_id == *parent_thread_id)
2308        {
2309            let identity = member
2310                .agent_path
2311                .as_deref()
2312                .or(member.task_name.as_deref())
2313                .unwrap_or(&member.name);
2314            let mut report = format!("Agent {identity} finished with status {status:?}.");
2315            if let Some(message) = final_message.as_deref()
2316                && !message.trim().is_empty()
2317            {
2318                report.push_str("\n\nFinal result:\n");
2319                report.push_str(message);
2320            }
2321            if let Some(error) = terminal_error.as_deref()
2322                && !error.trim().is_empty()
2323            {
2324                report.push_str("\n\nTerminal error:\n");
2325                report.push_str(error);
2326            }
2327            let mailbox_appended = self
2328                .teams
2329                .append_mailbox_message(
2330                    &team_id,
2331                    Some(member.id.clone()),
2332                    parent.id.clone(),
2333                    TeamMailboxMessageKind::FinalAnswer,
2334                    report,
2335                )
2336                .await
2337                .is_ok();
2338            if mailbox_appended
2339                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2340            {
2341                // The parent may finish between the active-turn lookup and enqueue. Its durable
2342                // mailbox entry remains pending for the next turn, while terminal observers must
2343                // still receive TeamMemberCompleted for this child.
2344                let _ = self
2345                    .deliver_pending_team_mailbox(
2346                        &team_id,
2347                        &parent.id,
2348                        parent_thread_id.clone(),
2349                        parent_turn_id,
2350                    )
2351                    .await;
2352            }
2353        }
2354        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2355            team_id,
2356            member_id: member.id,
2357            member_thread_id: member.thread_id,
2358            turn_id: Some(turn_id.clone()),
2359            status,
2360            final_message,
2361            error: terminal_error,
2362            timestamp: OffsetDateTime::now_utc(),
2363        }))
2364        .await;
2365        Ok(())
2366    }
2367
2368    /// Stops accepting new turns, requests interruption for every active turn,
2369    /// and waits for their runtime tasks to reach terminal cleanup up to
2370    /// `timeout`. Repeated calls are safe and continue draining the same set.
2371    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2372        let started_at = tokio::time::Instant::now();
2373        let persistence_failures_before =
2374            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2375        let turn_admission = self.turn_admission.lock().await;
2376        self.accepting_turns.store(false, Ordering::Release);
2377        let active = self
2378            .active_turns
2379            .read()
2380            .await
2381            .iter()
2382            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2383            .collect::<Vec<_>>();
2384        let draining = self
2385            .turn_drains
2386            .read()
2387            .await
2388            .iter()
2389            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2390            .collect::<Vec<_>>();
2391        drop(turn_admission);
2392
2393        let mut interrupted_turns = std::collections::BTreeMap::new();
2394        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2395            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2396        }
2397        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2398        for (turn_id, thread_id) in active {
2399            let _ = self
2400                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2401                .await;
2402        }
2403
2404        let wait_for_empty = async {
2405            loop {
2406                let notified = self.active_turns_changed.notified();
2407                if self.active_turns.read().await.is_empty()
2408                    && self.turn_drains.read().await.is_empty()
2409                {
2410                    return;
2411                }
2412                notified.await;
2413            }
2414        };
2415        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2416            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2417                > persistence_failures_before
2418            {
2419                RuntimeDrainOutcome::PersistenceFailed {
2420                    interrupted_turn_ids,
2421                    remaining_turn_ids: Vec::new(),
2422                }
2423            } else {
2424                RuntimeDrainOutcome::Clean {
2425                    interrupted_turn_ids,
2426                }
2427            }
2428        } else {
2429            let active_remaining = self
2430                .active_turns
2431                .read()
2432                .await
2433                .iter()
2434                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2435                .collect::<Vec<_>>();
2436            let draining_remaining = self
2437                .turn_drains
2438                .read()
2439                .await
2440                .iter()
2441                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2442                .collect::<Vec<_>>();
2443            let remaining = active_remaining
2444                .into_iter()
2445                .chain(draining_remaining)
2446                .collect::<std::collections::BTreeMap<_, _>>();
2447            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2448            for turn_id in &remaining_turn_ids {
2449                let handle = remaining
2450                    .get(turn_id)
2451                    .expect("remaining turn ID must have a drain handle");
2452                self.record_turn_lifecycle(
2453                    handle.thread_id.clone(),
2454                    turn_id.clone(),
2455                    TurnLifecycleState::InterruptRequested,
2456                    TurnCleanupState::TimedOut,
2457                    Some(TurnLifecycleReason::Shutdown),
2458                )
2459                .await;
2460            }
2461            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2462                > persistence_failures_before
2463            {
2464                RuntimeDrainOutcome::PersistenceFailed {
2465                    interrupted_turn_ids,
2466                    remaining_turn_ids,
2467                }
2468            } else {
2469                RuntimeDrainOutcome::DeadlineExceeded {
2470                    interrupted_turn_ids,
2471                    remaining_turn_ids,
2472                }
2473            }
2474        };
2475        self.record_lifecycle_drain_outcome(started_at, &outcome);
2476        outcome
2477    }
2478
2479    /// Enables turn submission after a prior drain attempt. This is intended
2480    /// for embedders that keep a runtime alive after a bounded drain timeout.
2481    pub fn resume_accepting_turns(&self) {
2482        self.accepting_turns.store(true, Ordering::Release);
2483    }
2484
2485    pub async fn turn_lifecycle_snapshot(
2486        &self,
2487        thread_id: &ThreadId,
2488    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2489        let Some(store) = &self.thread_store else {
2490            return Ok(TurnLifecycleSnapshot::default());
2491        };
2492        let extension_states = store.load_extension_states(thread_id).await?;
2493        Ok(turn_lifecycle_snapshot(&extension_states))
2494    }
2495
2496    pub async fn load_thread(
2497        &self,
2498        thread_id: &ThreadId,
2499    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2500        let loaded = if let Some(store) = &self.thread_store {
2501            store.load_thread(thread_id).await?
2502        } else {
2503            None
2504        };
2505        if let Some(snapshot) = loaded.as_ref() {
2506            let live_turn_ids = self
2507                .active_turns
2508                .read()
2509                .await
2510                .keys()
2511                .cloned()
2512                .chain(self.turn_drains.read().await.keys().cloned())
2513                .collect::<std::collections::HashSet<_>>();
2514            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2515            for record in lifecycle.records.into_iter().filter(|record| {
2516                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2517            }) {
2518                self.lifecycle_restart_reconciliations
2519                    .fetch_add(1, Ordering::AcqRel);
2520                self.record_turn_lifecycle(
2521                    record.thread_id,
2522                    record.turn_id,
2523                    TurnLifecycleState::RecoveryNeeded,
2524                    TurnCleanupState::Unknown,
2525                    Some(TurnLifecycleReason::RuntimeRestart),
2526                )
2527                .await;
2528            }
2529            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2530                thread_id: thread_id.clone(),
2531                timestamp: OffsetDateTime::now_utc(),
2532            }))
2533            .await;
2534        }
2535        Ok(loaded)
2536    }
2537
2538    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2539        if let Some(store) = &self.thread_store {
2540            let snapshot = store
2541                .load_thread(thread_id)
2542                .await?
2543                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2544            match snapshot {
2545                Ok(snapshot) => {
2546                    if let Some(metadata) = snapshot.metadata {
2547                        // Fork-backed threads fail closed before any write
2548                        // when their workspace was removed out-of-band.
2549                        if let Some(fork) = &metadata.workspace_fork
2550                            && fork.status == roder_api::forks::ForkStatus::Active
2551                            && !std::path::Path::new(&metadata.workspace).is_dir()
2552                        {
2553                            anyhow::bail!(
2554                                "workspace fork {} is missing its workspace at {}; restore it or \
2555                                 remove the fork before running turns in this thread",
2556                                fork.id,
2557                                metadata.workspace
2558                            );
2559                        }
2560                        return Ok(metadata.workspace);
2561                    }
2562                    eprintln!(
2563                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2564                    );
2565                }
2566                Err(err) => {
2567                    eprintln!(
2568                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2569                    );
2570                }
2571            }
2572        }
2573        Ok(self.workspace.display().to_string())
2574    }
2575
2576    async fn selection_mode_for_thread(
2577        &self,
2578        thread_id: &ThreadId,
2579    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2580        let Some(store) = &self.thread_store else {
2581            return Ok(None);
2582        };
2583        Ok(store
2584            .load_thread(thread_id)
2585            .await?
2586            .and_then(|snapshot| snapshot.metadata)
2587            .and_then(|metadata| {
2588                metadata
2589                    .selection_mode
2590                    .or_else(|| match (metadata.provider, metadata.model) {
2591                        (Some(provider), Some(model)) => {
2592                            Some(ModelSelectionMode::manual(provider, model, None))
2593                        }
2594                        _ => None,
2595                    })
2596            }))
2597    }
2598
2599    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2600    pub(crate) async fn thread_turn_overrides(
2601        &self,
2602        thread_id: &ThreadId,
2603    ) -> anyhow::Result<ThreadTurnOverrides> {
2604        let Some(store) = &self.thread_store else {
2605            return Ok(ThreadTurnOverrides::default());
2606        };
2607        Ok(store
2608            .load_thread_metadata(thread_id)
2609            .await?
2610            .map(|metadata| ThreadTurnOverrides {
2611                tool_allowlist: metadata.tool_allowlist,
2612                developer_instructions: metadata.developer_instructions,
2613                external_tools: metadata.external_tools,
2614            })
2615            .unwrap_or_default())
2616    }
2617
2618    pub async fn set_thread_selection_mode(
2619        &self,
2620        thread_id: &ThreadId,
2621        selection_mode: ModelSelectionMode,
2622    ) -> anyhow::Result<()> {
2623        let Some(store) = &self.thread_store else {
2624            return Ok(());
2625        };
2626        let Some(snapshot) = store.load_thread(thread_id).await? else {
2627            anyhow::bail!("thread not found: {thread_id}");
2628        };
2629        let Some(mut metadata) = snapshot.metadata else {
2630            return Ok(());
2631        };
2632        let concrete = selection_mode.concrete_selection();
2633        metadata.provider = Some(concrete.provider);
2634        metadata.model = Some(concrete.model);
2635        metadata.selection_mode = Some(selection_mode);
2636        metadata.updated_at = OffsetDateTime::now_utc();
2637        store.update_thread_metadata(metadata).await?;
2638        Ok(())
2639    }
2640
2641    async fn runner_session_for_thread(
2642        &self,
2643        thread_id: &ThreadId,
2644    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2645        let metadata = if let Some(store) = &self.thread_store {
2646            store.load_thread_metadata(thread_id).await?
2647        } else {
2648            None
2649        };
2650        // An explicit per-thread binding wins over the runtime-level destination.
2651        let destination = metadata
2652            .as_ref()
2653            .and_then(|metadata| metadata.runner_binding.as_ref())
2654            .map(|binding| binding.destination.clone())
2655            .or(self.config.read().await.remote_runner_destination.clone());
2656        let Some(destination) = destination else {
2657            self.evict_runner_session(thread_id).await;
2658            return Ok(None);
2659        };
2660        let provider = self
2661            .registry
2662            .remote_runner_providers
2663            .iter()
2664            .find(|provider| provider.id() == destination.provider_id)
2665            .cloned()
2666            .ok_or_else(|| {
2667                anyhow::anyhow!(
2668                    "remote runner provider {:?} is not installed",
2669                    destination.provider_id
2670                )
2671            })?;
2672        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2673        let slot = {
2674            let mut sessions = self.runner_sessions.lock().await;
2675            match sessions.get(thread_id) {
2676                Some(slot) if slot.matches(&destination) => slot.clone(),
2677                _ => {
2678                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2679                    sessions.insert(thread_id.clone(), slot.clone());
2680                    slot
2681                }
2682            }
2683        };
2684
2685        loop {
2686            let initialized = slot.initialized.notified();
2687            let mut state = slot.state.lock().await;
2688            match &*state {
2689                RunnerSessionSlotState::Ready(cached) => {
2690                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2691                }
2692                RunnerSessionSlotState::Failed(error) => {
2693                    return Err(anyhow::anyhow!(error.to_string()));
2694                }
2695                RunnerSessionSlotState::Initializing => {
2696                    drop(state);
2697                    initialized.await;
2698                }
2699                RunnerSessionSlotState::Empty => {
2700                    *state = RunnerSessionSlotState::Initializing;
2701                    drop(state);
2702                    self.spawn_runner_session_initialization(
2703                        thread_id.clone(),
2704                        destination.clone(),
2705                        provider.clone(),
2706                        persisted_state.clone(),
2707                        slot.clone(),
2708                    );
2709                }
2710            }
2711        }
2712    }
2713
2714    fn spawn_runner_session_initialization(
2715        &self,
2716        thread_id: ThreadId,
2717        destination: RunnerDestination,
2718        provider: Arc<dyn RemoteRunnerProvider>,
2719        persisted_state: Option<RunnerSessionState>,
2720        slot: Arc<RunnerSessionSlot>,
2721    ) {
2722        let thread_store = self.thread_store.clone();
2723        let runner_sessions = self.runner_sessions.clone();
2724        // The task intentionally outlives the turn that first requested the
2725        // workspace. Interrupting that turn must not strand the slot in
2726        // `Initializing` and deadlock every later tool or lifecycle call.
2727        drop(tokio::spawn(async move {
2728            let initialized = async {
2729                let session = if let Some(state) = persisted_state
2730                    && state.provider_id == destination.provider_id
2731                    && state.destination_id == destination.id
2732                {
2733                    match provider.resume_session(state).await {
2734                        Ok(session) => session,
2735                        Err(_) => provider.create_session(destination.clone()).await?,
2736                    }
2737                } else {
2738                    provider.create_session(destination.clone()).await?
2739                };
2740                let resolved = (destination.clone(), session.clone());
2741                // Persist before releasing the singleflight or dispatching
2742                // the first tool. A later process can rejoin even if the turn
2743                // that requested initialization has already been interrupted.
2744                Self::persist_runner_state_in_store(
2745                    thread_store.as_ref(),
2746                    &thread_id,
2747                    Some(&resolved),
2748                )
2749                .await?;
2750                Ok::<_, anyhow::Error>(CachedRunnerSession {
2751                    destination,
2752                    session,
2753                })
2754            }
2755            .await;
2756
2757            match initialized {
2758                Ok(cached) => {
2759                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2760                    slot.initialized.notify_waiters();
2761                }
2762                Err(error) => {
2763                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2764                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2765                    slot.initialized.notify_waiters();
2766                    let mut sessions = runner_sessions.lock().await;
2767                    let is_current = sessions
2768                        .get(&thread_id)
2769                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2770                    if is_current {
2771                        sessions.remove(&thread_id);
2772                    }
2773                }
2774            }
2775        }));
2776    }
2777
2778    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2779        self.runner_sessions.lock().await.remove(thread_id);
2780    }
2781
2782    async fn cache_runner_session(
2783        &self,
2784        thread_id: &ThreadId,
2785        destination: RunnerDestination,
2786        session: Arc<dyn RemoteRunnerSession>,
2787    ) {
2788        let cached = CachedRunnerSession {
2789            destination,
2790            session,
2791        };
2792        self.runner_sessions.lock().await.insert(
2793            thread_id.clone(),
2794            Arc::new(RunnerSessionSlot::ready(cached)),
2795        );
2796    }
2797
2798    /// Read a thread's explicit runner binding without creating or resuming a
2799    /// session. Tool routing uses this before local previews so runner-backed
2800    /// and fail-closed hosted calls never inspect the host workspace.
2801    pub(crate) async fn runner_binding_for_thread(
2802        &self,
2803        thread_id: &ThreadId,
2804    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2805        let Some(store) = &self.thread_store else {
2806            return Ok(None);
2807        };
2808        Ok(store
2809            .load_thread_metadata(thread_id)
2810            .await?
2811            .and_then(|metadata| metadata.runner_binding))
2812    }
2813
2814    /// Resolve a previously-read binding into a live remote workspace. The
2815    /// session remains lazy: callers invoke this only after policy approval.
2816    pub(crate) async fn remote_workspace_for_binding(
2817        &self,
2818        thread_id: &ThreadId,
2819        binding: ThreadRunnerBinding,
2820    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2821        let session = self
2822            .runner_session_for_thread(thread_id)
2823            .await?
2824            .map(|(_, session)| session)
2825            .ok_or_else(|| {
2826                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2827            })?;
2828        Ok(Arc::new(RemoteWorkspace {
2829            session,
2830            root: binding.workspace,
2831            read_roots: binding.read_roots,
2832        }))
2833    }
2834
2835    async fn persist_runner_state(
2836        &self,
2837        thread_id: &ThreadId,
2838        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2839    ) -> anyhow::Result<()> {
2840        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2841    }
2842
2843    async fn persist_runner_state_in_store(
2844        store: Option<&Arc<dyn ThreadStore>>,
2845        thread_id: &ThreadId,
2846        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2847    ) -> anyhow::Result<()> {
2848        let Some((destination, session)) = runner else {
2849            return Ok(());
2850        };
2851        let Some(store) = store else {
2852            return Ok(());
2853        };
2854        let Some(snapshot) = store.load_thread(thread_id).await? else {
2855            return Ok(());
2856        };
2857        let Some(mut metadata) = snapshot.metadata else {
2858            return Ok(());
2859        };
2860        metadata.runner_destination = Some(destination.clone());
2861        metadata.runner_state = Some(session.state());
2862        metadata.updated_at = OffsetDateTime::now_utc();
2863        store.update_thread_metadata(metadata).await?;
2864        Ok(())
2865    }
2866
2867    fn remote_runner_provider_by_id(
2868        &self,
2869        provider_id: &str,
2870    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2871        self.registry
2872            .remote_runner_providers
2873            .iter()
2874            .find(|provider| provider.id() == provider_id)
2875            .cloned()
2876    }
2877
2878    /// Resolve the live runner session for a thread along with its provider,
2879    /// failing clearly when the thread is not runner-bound.
2880    async fn thread_runner_session(
2881        &self,
2882        thread_id: &ThreadId,
2883    ) -> anyhow::Result<(
2884        RunnerDestination,
2885        Arc<dyn RemoteRunnerProvider>,
2886        Arc<dyn RemoteRunnerSession>,
2887    )> {
2888        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2889            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2890        };
2891        let provider = self
2892            .remote_runner_provider_by_id(&destination.provider_id)
2893            .ok_or_else(|| {
2894                anyhow::anyhow!(
2895                    "remote runner provider {:?} is not installed",
2896                    destination.provider_id
2897                )
2898            })?;
2899        Ok((destination, provider, session))
2900    }
2901
2902    /// Pause a runner-bound thread's session toward standby and persist the
2903    /// post-pause state. Errors if the provider is not pausable.
2904    pub async fn pause_thread_runner(
2905        &self,
2906        thread_id: &ThreadId,
2907    ) -> anyhow::Result<RunnerSessionState> {
2908        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2909        anyhow::ensure!(
2910            provider.capabilities().pausable,
2911            "remote runner provider {:?} does not support pausing",
2912            destination.provider_id
2913        );
2914        let state = session.pause().await?;
2915        self.persist_runner_state(thread_id, Some(&(destination, session)))
2916            .await?;
2917        Ok(state)
2918    }
2919
2920    /// Resume (wake) a runner-bound thread's paused session and persist state.
2921    pub async fn resume_thread_runner(
2922        &self,
2923        thread_id: &ThreadId,
2924    ) -> anyhow::Result<RunnerSessionState> {
2925        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
2926        let state = session.resume().await?;
2927        self.persist_runner_state(thread_id, Some(&(destination, session)))
2928            .await?;
2929        Ok(state)
2930    }
2931
2932    /// Detach a runner-bound thread's session: persist the durable, rejoinable
2933    /// state and leave the remote sandbox alive. Errors if not detachable.
2934    pub async fn detach_thread_runner(
2935        &self,
2936        thread_id: &ThreadId,
2937    ) -> anyhow::Result<RunnerSessionState> {
2938        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2939        anyhow::ensure!(
2940            provider.capabilities().detachable,
2941            "remote runner provider {:?} does not support detaching",
2942            destination.provider_id
2943        );
2944        let state = session.detach().await?;
2945        // Persist the detached state explicitly so a later turn or process
2946        // rejoins the same sandbox instead of provisioning a new one.
2947        if let Some(store) = &self.thread_store
2948            && let Some(snapshot) = store.load_thread(thread_id).await?
2949            && let Some(mut metadata) = snapshot.metadata
2950        {
2951            metadata.runner_destination = Some(destination);
2952            metadata.runner_state = Some(state.clone());
2953            metadata.updated_at = OffsetDateTime::now_utc();
2954            store.update_thread_metadata(metadata).await?;
2955        }
2956        self.evict_runner_session(thread_id).await;
2957        Ok(state)
2958    }
2959
2960    /// Rejoin a previously created sandbox from a thread's persisted runner
2961    /// state without provisioning a new one. An optional `sandbox` overrides the
2962    /// persisted sandbox name (recovery by name). Persists refreshed state.
2963    pub async fn rejoin_thread_runner(
2964        &self,
2965        thread_id: &ThreadId,
2966        sandbox: Option<String>,
2967    ) -> anyhow::Result<RunnerSessionState> {
2968        let store = self
2969            .thread_store
2970            .as_ref()
2971            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
2972        let metadata = store
2973            .load_thread_metadata(thread_id)
2974            .await?
2975            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
2976        let destination = metadata
2977            .runner_binding
2978            .as_ref()
2979            .map(|binding| binding.destination.clone())
2980            .or_else(|| metadata.runner_destination.clone())
2981            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
2982        let mut state = metadata
2983            .runner_state
2984            .clone()
2985            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
2986        if let Some(sandbox) = sandbox
2987            && let Some(object) = state.metadata.as_object_mut()
2988        {
2989            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
2990        }
2991        let provider = self
2992            .remote_runner_provider_by_id(&destination.provider_id)
2993            .ok_or_else(|| {
2994                anyhow::anyhow!(
2995                    "remote runner provider {:?} is not installed",
2996                    destination.provider_id
2997                )
2998            })?;
2999        let session = provider.rejoin_session(state).await?;
3000        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3001            .await?;
3002        self.cache_runner_session(thread_id, destination, session.clone())
3003            .await;
3004        Ok(session.state())
3005    }
3006
3007    async fn record_thread_usage_metadata(
3008        &self,
3009        thread_id: &ThreadId,
3010        usage: &TokenUsage,
3011    ) -> anyhow::Result<()> {
3012        if usage.is_empty() {
3013            return Ok(());
3014        }
3015        let Some(store) = &self.thread_store else {
3016            return Ok(());
3017        };
3018        let Some(snapshot) = store.load_thread(thread_id).await? else {
3019            return Ok(());
3020        };
3021        let Some(mut metadata) = snapshot.metadata else {
3022            return Ok(());
3023        };
3024        metadata
3025            .usage
3026            .get_or_insert_with(ThreadUsageMetadata::default)
3027            .add_token_usage(usage);
3028        metadata.updated_at = OffsetDateTime::now_utc();
3029        store.update_thread_metadata(metadata).await?;
3030        Ok(())
3031    }
3032
3033    pub fn start_turn(
3034        self: &Arc<Self>,
3035        mut req: StartTurnRequest,
3036    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3037        Box::pin(async move {
3038            let turn_admission = self.turn_admission.lock().await;
3039            anyhow::ensure!(
3040                self.accepting_turns.load(Ordering::Acquire),
3041                "runtime is quiescing and cannot accept new turns"
3042            );
3043            req.workspace = validate_thread_workspace(&req.workspace)?;
3044            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3045            let cfg = self.config.read().await.clone();
3046            let provider = req
3047                .provider_override
3048                .clone()
3049                .unwrap_or_else(|| cfg.default_provider.clone());
3050            self.engine_for(&provider)?;
3051            let turn_id = uuid::Uuid::new_v4().to_string();
3052            let mut initial_mailbox_ack = None;
3053            if let Some((team_id, member)) = &team_member {
3054                let pending = self
3055                    .teams
3056                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3057                    .await?;
3058                if !pending.is_empty()
3059                    && let Some(team) = self.read_team(team_id).await
3060                {
3061                    let mailbox = format_mailbox_messages(&team, &pending);
3062                    req.message = if req.message.trim().is_empty() {
3063                        mailbox
3064                    } else {
3065                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3066                    };
3067                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3068                        team_id: team_id.clone(),
3069                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3070                    });
3071                }
3072            }
3073            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3074            let drain = Arc::new(TurnDrainHandle {
3075                thread_id: req.thread_id.clone(),
3076                interrupt_requested: AtomicBool::new(false),
3077                interrupt_reason: Mutex::new(None),
3078                completed: AtomicBool::new(false),
3079                completed_notify: Notify::new(),
3080            });
3081            let active = ActiveTurnHandle {
3082                thread_id: req.thread_id.clone(),
3083                abort: abort_handle,
3084                steers: Arc::new(Mutex::new(Vec::new())),
3085                drain,
3086            };
3087            self.active_turns
3088                .write()
3089                .await
3090                .insert(turn_id.clone(), active);
3091            self.record_turn_lifecycle(
3092                req.thread_id.clone(),
3093                turn_id.clone(),
3094                TurnLifecycleState::Running,
3095                TurnCleanupState::NotRequested,
3096                None,
3097            )
3098            .await;
3099            self.active_turn_contexts.write().await.insert(
3100                turn_id.clone(),
3101                InheritedTurnContext {
3102                    workspace: req.workspace.clone(),
3103                    instructions: req.instructions.clone(),
3104                    developer_context: req.developer_context.clone(),
3105                },
3106            );
3107            if let Some((team_id, member)) = team_member {
3108                let updated = match self
3109                    .teams
3110                    .update_member(&team_id, &member.id, |member| {
3111                        member.current_turn_id = Some(turn_id.clone());
3112                        member.status = TeamMemberStatus::Running;
3113                        member.final_message = None;
3114                        member.terminal_error = None;
3115                    })
3116                    .await
3117                {
3118                    Ok(updated) => updated,
3119                    Err(error) => {
3120                        self.active_turns.write().await.remove(&turn_id);
3121                        self.active_turn_contexts.write().await.remove(&turn_id);
3122                        self.teams
3123                            .release_mailbox_reservations_for_turn(&turn_id)
3124                            .await;
3125                        return Err(error);
3126                    }
3127                };
3128                if let Some(member) = updated
3129                    .members
3130                    .into_iter()
3131                    .find(|candidate| candidate.id == member.id)
3132                {
3133                    self.emit(RoderEvent::TeamMemberStatusChanged(
3134                        TeamMemberStatusChanged {
3135                            team_id,
3136                            member_id: member.id,
3137                            member_thread_id: member.thread_id,
3138                            status: TeamMemberStatus::Running,
3139                            timestamp: OffsetDateTime::now_utc(),
3140                        },
3141                    ))
3142                    .await;
3143                }
3144            }
3145            let runtime = Arc::clone(self);
3146            let turn_req = req;
3147            let thread_id_for_task = turn_req.thread_id.clone();
3148            let turn_id_for_task = turn_id.clone();
3149            tokio::spawn(async move {
3150                let result = Abortable::new(
3151                    runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3152                    abort_registration,
3153                )
3154                .await;
3155                /*
3156                 * A failed sibling in a parallel tool batch drops in-flight external tool
3157                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3158                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3159                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3160                 * resolution; on clean completion the map holds nothing for this turn.
3161                 */
3162                runtime
3163                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3164                    .await;
3165                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3166                match &result {
3167                    Ok(Err(err)) => {
3168                        let (cleanup, ownership) =
3169                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3170                        // run_turn emits failures after the stream starts; this covers setup/startup errors.
3171                        runtime
3172                            .emit(RoderEvent::TurnFailed(TurnFailed {
3173                                thread_id: thread_id_for_task.clone(),
3174                                turn_id: turn_id_for_task.clone(),
3175                                error: err.to_string(),
3176                                error_kind: None,
3177                                usage: None,
3178                                timestamp: OffsetDateTime::now_utc(),
3179                            }))
3180                            .await;
3181                        runtime
3182                            .record_turn_lifecycle_with_ownership(
3183                                thread_id_for_task.clone(),
3184                                turn_id_for_task.clone(),
3185                                TurnLifecycleState::Failed,
3186                                cleanup,
3187                                Some(TurnLifecycleReason::ProviderFailure),
3188                                ownership,
3189                            )
3190                            .await;
3191                        let _ = runtime
3192                            .complete_team_member_turn_with_result(
3193                                &thread_id_for_task,
3194                                &turn_id_for_task,
3195                                TeamMemberStatus::Failed,
3196                                None,
3197                                Some(err.to_string()),
3198                            )
3199                            .await;
3200                    }
3201                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3202                        let (cleanup, ownership) =
3203                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3204                        runtime
3205                            .record_turn_lifecycle_with_ownership(
3206                                thread_id_for_task.clone(),
3207                                turn_id_for_task.clone(),
3208                                TurnLifecycleState::Failed,
3209                                cleanup,
3210                                Some(TurnLifecycleReason::ProviderFailure),
3211                                ownership,
3212                            )
3213                            .await;
3214                        let _ = runtime
3215                            .complete_team_member_turn_with_result(
3216                                &thread_id_for_task,
3217                                &turn_id_for_task,
3218                                TeamMemberStatus::Failed,
3219                                None,
3220                                Some("turn stopped before completion".to_string()),
3221                            )
3222                            .await;
3223                    }
3224                    Err(_) => {
3225                        let reason = if let Some(handle) = runtime
3226                            .turn_drains
3227                            .read()
3228                            .await
3229                            .get(&turn_id_for_task)
3230                            .cloned()
3231                        {
3232                            handle
3233                                .interrupt_reason
3234                                .lock()
3235                                .await
3236                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3237                        } else {
3238                            TurnLifecycleReason::RuntimeFailure
3239                        };
3240                        let (cleanup, ownership) =
3241                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3242                        runtime
3243                            .record_turn_lifecycle_with_ownership(
3244                                thread_id_for_task.clone(),
3245                                turn_id_for_task.clone(),
3246                                TurnLifecycleState::Interrupted,
3247                                cleanup,
3248                                Some(reason),
3249                                ownership,
3250                            )
3251                            .await;
3252                        runtime
3253                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3254                                thread_id: thread_id_for_task.clone(),
3255                                turn_id: turn_id_for_task.clone(),
3256                                timestamp: OffsetDateTime::now_utc(),
3257                            }))
3258                            .await;
3259                        let _ = runtime
3260                            .complete_team_member_turn_with_result(
3261                                &thread_id_for_task,
3262                                &turn_id_for_task,
3263                                TeamMemberStatus::Interrupted,
3264                                None,
3265                                None,
3266                            )
3267                            .await;
3268                    }
3269                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3270                }
3271                runtime
3272                    .teams
3273                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3274                    .await;
3275                runtime.active_turns.write().await.remove(&turn_id_for_task);
3276                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3277                    drain.completed.store(true, Ordering::Release);
3278                    drain.completed_notify.notify_waiters();
3279                }
3280                runtime
3281                    .active_turn_selections
3282                    .write()
3283                    .await
3284                    .remove(&turn_id_for_task);
3285                runtime
3286                    .active_turn_contexts
3287                    .write()
3288                    .await
3289                    .remove(&turn_id_for_task);
3290                if !completed {
3291                    // Completion paths above consume registered provider cleanup
3292                    // handles. A setup failure before an engine can stream has no
3293                    // such handle; this is a harmless final defensive sweep.
3294                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3295                        cleanups.remove(&turn_id_for_task);
3296                    });
3297                }
3298                runtime.active_turns_changed.notify_waiters();
3299                if completed {
3300                    let _ = runtime
3301                        .continue_active_goal_after_turn(thread_id_for_task)
3302                        .await;
3303                }
3304            });
3305            drop(turn_admission);
3306            Ok(turn_id)
3307        })
3308    }
3309
3310    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3311        self.active_turns
3312            .read()
3313            .await
3314            .values()
3315            .any(|handle| &handle.thread_id == thread_id)
3316    }
3317
3318    async fn ensure_codex_v2_team_capacity(
3319        &self,
3320        team: &TeamState,
3321        resuming_member_id: &str,
3322        candidate_model: Option<&str>,
3323    ) -> anyhow::Result<()> {
3324        if !is_codex_v2_team(team)
3325            && !candidate_model
3326                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3327        {
3328            return Ok(());
3329        }
3330        let active_thread_ids = self
3331            .active_turns
3332            .read()
3333            .await
3334            .values()
3335            .map(|handle| handle.thread_id.clone())
3336            .collect::<std::collections::HashSet<_>>();
3337        let resident_threads = 1 + team
3338            .members
3339            .iter()
3340            .filter(|member| {
3341                member.role != roder_api::teams::TeamMemberRole::Lead
3342                    && member.id != resuming_member_id
3343                    && active_thread_ids.contains(&member.thread_id)
3344            })
3345            .count();
3346        anyhow::ensure!(
3347            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3348            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3349            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3350        );
3351        Ok(())
3352    }
3353
3354    /// Number of currently running turns across all threads. Hosted runtime
3355    /// pools use this to avoid evicting tenants with active work.
3356    pub async fn active_turn_count(&self) -> usize {
3357        self.active_turns.read().await.len()
3358    }
3359
3360    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3361        self.active_turns
3362            .read()
3363            .await
3364            .iter()
3365            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3366    }
3367
3368    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3369        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3370        let draining_turn_id = if active_turn_id.is_none() {
3371            self.turn_drains
3372                .read()
3373                .await
3374                .iter()
3375                .find_map(|(turn_id, drain)| {
3376                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3377                })
3378        } else {
3379            None
3380        };
3381        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3382            return ThreadActivity::default();
3383        };
3384
3385        let mut active_flags = Vec::new();
3386        {
3387            let pending_approvals = self.pending_tool_approvals.lock().await;
3388            if pending_approvals
3389                .values()
3390                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3391            {
3392                active_flags.push("approvalRequired".to_string());
3393            }
3394        }
3395        {
3396            let pending_inputs = self.pending_user_inputs.lock().await;
3397            if pending_inputs
3398                .values()
3399                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3400            {
3401                active_flags.push("userInputRequired".to_string());
3402            }
3403        }
3404        {
3405            let pending_external = self.pending_external_tool_calls.lock().await;
3406            if pending_external
3407                .values()
3408                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3409            {
3410                active_flags.push("externalToolPending".to_string());
3411            }
3412        }
3413        let interrupting = self
3414            .active_turns
3415            .read()
3416            .await
3417            .get(&active_turn_id)
3418            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3419            || self
3420                .turn_drains
3421                .read()
3422                .await
3423                .get(&active_turn_id)
3424                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3425        if interrupting {
3426            active_flags.push("interrupting".to_string());
3427        }
3428        if self.pending_plan_exit().await.is_some_and(|pending| {
3429            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3430        }) {
3431            active_flags.push("planExitRequired".to_string());
3432        }
3433
3434        ThreadActivity {
3435            active_turn_id: Some(active_turn_id),
3436            active_flags,
3437        }
3438    }
3439
3440    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3441        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3442            .await
3443    }
3444
3445    async fn interrupt_turn_with_reason(
3446        &self,
3447        thread_id: ThreadId,
3448        turn_id: TurnId,
3449        reason: TurnLifecycleReason,
3450    ) -> anyhow::Result<()> {
3451        let handle = self.active_turns.write().await.remove(&turn_id);
3452        if let Some(handle) = handle {
3453            if handle
3454                .drain
3455                .interrupt_requested
3456                .swap(true, Ordering::AcqRel)
3457            {
3458                return Ok(());
3459            }
3460            *handle.drain.interrupt_reason.lock().await = Some(reason);
3461            self.turn_drains
3462                .write()
3463                .await
3464                .insert(turn_id.clone(), handle.drain.clone());
3465            let ownership = self.provider_cleanup_ownership(&turn_id);
3466            self.record_turn_lifecycle_with_ownership(
3467                thread_id.clone(),
3468                turn_id.clone(),
3469                TurnLifecycleState::InterruptRequested,
3470                TurnCleanupState::Requested,
3471                Some(reason),
3472                ownership,
3473            )
3474            .await;
3475            handle.abort.abort();
3476            self.active_turns_changed.notify_waiters();
3477        }
3478        self.active_turn_selections.write().await.remove(&turn_id);
3479        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3480            .await;
3481        Ok(())
3482    }
3483
3484    pub async fn steer_turn(
3485        &self,
3486        thread_id: ThreadId,
3487        turn_id: TurnId,
3488        message: String,
3489        images: Vec<InputImage>,
3490    ) -> anyhow::Result<()> {
3491        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3492            .await
3493    }
3494
3495    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3496        let active = self.active_turns.read().await.get(turn_id).cloned();
3497        let Some(active) = active else {
3498            return false;
3499        };
3500        let has_pending = !active.steers.lock().await.is_empty();
3501        has_pending
3502    }
3503
3504    async fn steer_turn_with_mailbox_ack(
3505        &self,
3506        thread_id: ThreadId,
3507        turn_id: TurnId,
3508        message: String,
3509        message_ids: Vec<String>,
3510        team_id: TeamId,
3511    ) -> anyhow::Result<()> {
3512        self.enqueue_turn_steer(
3513            thread_id,
3514            turn_id,
3515            message,
3516            Vec::new(),
3517            Some(MailboxDeliveryAck {
3518                team_id,
3519                message_ids,
3520            }),
3521        )
3522        .await
3523    }
3524
3525    async fn deliver_pending_team_mailbox(
3526        &self,
3527        team_id: &str,
3528        member_id: &str,
3529        member_thread_id: ThreadId,
3530        turn_id: TurnId,
3531    ) -> anyhow::Result<()> {
3532        let pending = self
3533            .teams
3534            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3535            .await?;
3536        if pending.is_empty() {
3537            return Ok(());
3538        }
3539        let message_ids = pending
3540            .iter()
3541            .map(|message| message.id.clone())
3542            .collect::<Vec<_>>();
3543        let team = self
3544            .read_team(team_id)
3545            .await
3546            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3547        if let Err(error) = self
3548            .steer_turn_with_mailbox_ack(
3549                member_thread_id,
3550                turn_id.clone(),
3551                format_mailbox_messages(&team, &pending),
3552                message_ids.clone(),
3553                team_id.to_string(),
3554            )
3555            .await
3556        {
3557            self.teams
3558                .release_mailbox_reservations(&turn_id, &message_ids)
3559                .await;
3560            return Err(error);
3561        }
3562        Ok(())
3563    }
3564
3565    async fn enqueue_turn_steer(
3566        &self,
3567        thread_id: ThreadId,
3568        turn_id: TurnId,
3569        message: String,
3570        images: Vec<InputImage>,
3571        mailbox_ack: Option<MailboxDeliveryAck>,
3572    ) -> anyhow::Result<()> {
3573        let message = message.trim().to_string();
3574        if message.is_empty() && images.is_empty() {
3575            return Ok(());
3576        }
3577
3578        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3579            anyhow::bail!("no active turn to steer");
3580        };
3581        active.steers.lock().await.push(QueuedTurnSteer {
3582            message: UserMessage::with_images(message.clone(), images),
3583            mailbox_ack,
3584        });
3585        self.emit(RoderEvent::TurnSteered(TurnSteered {
3586            thread_id,
3587            turn_id,
3588            message,
3589            timestamp: OffsetDateTime::now_utc(),
3590        }))
3591        .await;
3592        Ok(())
3593    }
3594
3595    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3596        let cfg = self.config.read().await;
3597        let model_profile =
3598            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3599        self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3600    }
3601
3602    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3603        self.registry
3604            .subagent_dispatchers
3605            .iter()
3606            .flat_map(|dispatcher| dispatcher.definitions())
3607            .collect()
3608    }
3609
3610    async fn run_turn(
3611        self: &Arc<Self>,
3612        req: StartTurnRequest,
3613        turn_id: TurnId,
3614        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3615    ) -> anyhow::Result<TurnRunOutcome> {
3616        let turn_started_at = OffsetDateTime::now_utc();
3617        self.emit(RoderEvent::TurnStarted(TurnStarted {
3618            thread_id: req.thread_id.clone(),
3619            turn_id: turn_id.clone(),
3620            runtime_profile: self.config.read().await.runtime_profile,
3621            timestamp: turn_started_at,
3622        }))
3623        .await;
3624        self.persist_turn_item(
3625            &req.thread_id,
3626            &turn_id,
3627            &TranscriptItem::UserMessage(UserMessage::with_images(
3628                req.message.clone(),
3629                req.images.clone(),
3630            )),
3631        )
3632        .await?;
3633        if let Some(ack) = initial_mailbox_ack {
3634            self.teams
3635                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3636                .await?;
3637        }
3638
3639        let mut cfg = self.config.read().await.clone();
3640        let runtime_profile = cfg.runtime_profile;
3641        let turn_deadline = turn_deadline_for_config(&cfg);
3642        let deadline_finalization_reserve =
3643            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3644        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3645        let concrete_selection = selection_mode
3646            .as_ref()
3647            .map(ModelSelectionMode::concrete_selection);
3648        let default_provider = req
3649            .provider_override
3650            .clone()
3651            .or_else(|| {
3652                concrete_selection
3653                    .as_ref()
3654                    .map(|selection| selection.provider.clone())
3655            })
3656            .unwrap_or(cfg.default_provider.clone());
3657        let default_model = req
3658            .model_override
3659            .clone()
3660            .or_else(|| {
3661                concrete_selection
3662                    .as_ref()
3663                    .map(|selection| selection.model.clone())
3664            })
3665            .unwrap_or(cfg.default_model.clone());
3666        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3667            selection_mode
3668                .as_ref()
3669                .and_then(ModelSelectionMode::reasoning)
3670        }) {
3671            validate_reasoning_effort(&default_model, reasoning)?;
3672            cfg.reasoning = Some(reasoning.to_string());
3673        }
3674        let turn_has_concrete_model_override =
3675            req.provider_override.is_some() || req.model_override.is_some();
3676        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3677            Some(ModelSelectionMode::Auto {
3678                router_id, profile, ..
3679            }) if !turn_has_concrete_model_override => (
3680                RuntimeInferenceRouterConfig {
3681                    enabled: true,
3682                    router_id: Some(router_id.clone()),
3683                },
3684                profile.clone(),
3685            ),
3686            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3687        };
3688        let mut provider = default_provider.clone();
3689        let mut model = default_model.clone();
3690        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3691        let workspace = req.workspace.clone();
3692        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3693        let mut compacted_this_turn = transcript
3694            .iter()
3695            .any(crate::compaction::is_compaction_boundary);
3696        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3697        let agent_swarm_mode_active = self
3698            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3699            .await;
3700        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3701        let mut final_assistant_text = String::new();
3702        let mut final_phase_messages = Vec::<AssistantMessage>::new();
3703        let mut final_reasoning_text = String::new();
3704        let mut final_provider_metadata = None;
3705        let mut exhausted_tool_rounds = true;
3706        let mut verification_gate =
3707            VerificationGateState::new(req.message.clone(), runtime_profile);
3708        let mut speed_policy = SpeedPolicyState::default();
3709        let mut reliability = TurnReliabilityState::default();
3710        let mut turn_usage = TokenUsage::default();
3711        // Overwritten on every inference step's Completed event so only the
3712        // terminal step's stop reason survives (mid-turn tool_use steps must
3713        // not leak onto turn/completed).
3714        let mut turn_finish_reason: Option<String> = None;
3715        let mut deadline_finalization_requested = false;
3716        let mut deadline_scoreable_completion_requested = false;
3717        let mut task_ledger_completion_reminders = 0_u8;
3718        let mut task_ledger_scoreable_checkpoints = 0_u8;
3719        let mut empty_tool_call_nudges_used = 0_u32;
3720        let mut provider_stream_retry_attempts = 0_u32;
3721        let mut routing_candidates = None;
3722        let routing_transcript_start = transcript.len().saturating_sub(1);
3723        let mut routing_escalations = 0_u32;
3724        let mut model_switch_summary_selection = None::<ModelSelection>;
3725
3726        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3727            if let Some(deadline) = turn_deadline
3728                && deadline_expired(deadline)
3729            {
3730                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3731                    .await?;
3732                return Ok(TurnRunOutcome::Stopped);
3733            }
3734            let steers = self.drain_turn_steers(&turn_id).await;
3735            self.append_steers(&req, &turn_id, &mut transcript, steers)
3736                .await?;
3737            if runtime_profile == RuntimeProfile::Eval
3738                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3739                    turn_deadline,
3740                    deadline_finalization_reserve,
3741                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3742                )
3743            {
3744                if req.task_ledger_required
3745                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3746                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3747                {
3748                    task_ledger_completion_reminders += 1;
3749                    deadline_scoreable_completion_requested = true;
3750                    let item = TranscriptItem::UserMessage(UserMessage::text(
3751                        task_ledger_deadline_completion_prompt(
3752                            remaining,
3753                            deadline_finalization_reserve,
3754                            &prompt,
3755                        ),
3756                    ));
3757                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3758                        .await?;
3759                    transcript.push(item);
3760                    continue 'tool_rounds;
3761                } else {
3762                    self.start_deadline_finalization(
3763                        &req.thread_id,
3764                        &turn_id,
3765                        &mut transcript,
3766                        remaining,
3767                    )
3768                    .await?;
3769                    deadline_finalization_requested = true;
3770                }
3771            }
3772            if runtime_profile == RuntimeProfile::Eval
3773                && req.task_ledger_required
3774                && !deadline_finalization_requested
3775                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3776                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3777                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3778                && remaining > deadline_finalization_reserve
3779                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3780            {
3781                task_ledger_scoreable_checkpoints += 1;
3782                let item = TranscriptItem::UserMessage(UserMessage::text(
3783                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3784                ));
3785                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3786                    .await?;
3787                transcript.push(item);
3788                continue 'tool_rounds;
3789            }
3790            if turn_inference_router.is_active() && routing_candidates.is_none() {
3791                routing_candidates =
3792                    Some(collect_inference_routing_candidates(&self.registry).await);
3793            }
3794            let routing_tools_model = model.clone();
3795            let routing_tools = self.filtered_tool_specs(
3796                &cfg,
3797                &model,
3798                model_profile.as_ref(),
3799                &thread_overrides.tool_allowlist,
3800                &thread_overrides.external_tools,
3801            );
3802            let prior_failures =
3803                transcript_failure_count_since(&transcript, routing_transcript_start)
3804                    .max(reliability.tool_failure_count())
3805                    .saturating_add(provider_stream_retry_attempts);
3806            let routing_selection = route_inference_selection(
3807                &self.registry,
3808                &turn_inference_router,
3809                InferenceRoutingRequest {
3810                    thread_id: &req.thread_id,
3811                    turn_id: &turn_id,
3812                    round_index: round_index as u32,
3813                    runtime_profile,
3814                    phase: speed_policy.phase(),
3815                    profile: turn_inference_router_profile.as_deref(),
3816                    default_selection: ModelSelection {
3817                        provider: default_provider.clone(),
3818                        model: default_model.clone(),
3819                    },
3820                    transcript: &transcript,
3821                    tools: &routing_tools,
3822                    candidates: routing_candidates.as_deref(),
3823                    prior_failures,
3824                    prior_escalations: routing_escalations,
3825                },
3826            )
3827            .await;
3828            if let Some(decision) = routing_selection.decision.clone() {
3829                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3830                    routing_escalations = routing_escalations.saturating_add(1);
3831                }
3832                self.emit(RoderEvent::InferenceRoutingDecision(
3833                    InferenceRoutingDecisionEvent {
3834                        thread_id: req.thread_id.clone(),
3835                        turn_id: turn_id.clone(),
3836                        round_index: round_index as u32,
3837                        default_selection: ModelSelection {
3838                            provider: default_provider.clone(),
3839                            model: default_model.clone(),
3840                        },
3841                        selected_selection: routing_selection.selection.clone(),
3842                        decision,
3843                        timestamp: OffsetDateTime::now_utc(),
3844                    },
3845                ))
3846                .await;
3847            }
3848            provider = routing_selection.selection.provider.clone();
3849            model = routing_selection.selection.model.clone();
3850            let engine = self.engine_for(&provider)?;
3851            let capabilities = engine.capabilities();
3852            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3853            let tools = if capabilities.tool_calls {
3854                if model == routing_tools_model {
3855                    routing_tools.clone()
3856                } else {
3857                    self.filtered_tool_specs(
3858                        &cfg,
3859                        &model,
3860                        model_profile.as_ref(),
3861                        &thread_overrides.tool_allowlist,
3862                        &thread_overrides.external_tools,
3863                    )
3864                }
3865            } else {
3866                Vec::new()
3867            };
3868            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3869            let tool_choice = if tools.is_empty() {
3870                ToolChoice::None
3871            } else {
3872                ToolChoice::Auto
3873            };
3874            let summary_selection = ModelSelection {
3875                provider: provider.clone(),
3876                model: model.clone(),
3877            };
3878            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3879                if let Some(summary) = model_switch_summary(
3880                    &transcript,
3881                    model_profile.as_ref(),
3882                    &provider,
3883                    &model,
3884                    &tools,
3885                ) {
3886                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3887                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3888                        .await?;
3889                    transcript.push(item);
3890                }
3891                model_switch_summary_selection = Some(summary_selection);
3892            }
3893
3894            if !capabilities.image_input && transcript_has_images(&transcript) {
3895                self.fail_turn_with_error(
3896                    &req.thread_id,
3897                    &turn_id,
3898                    format!("provider {provider} does not support image input"),
3899                )
3900                .await?;
3901                return Ok(TurnRunOutcome::Stopped);
3902            }
3903            transcript = self
3904                .compact_transcript_if_needed(
3905                    &req.thread_id,
3906                    &turn_id,
3907                    &provider,
3908                    &model,
3909                    transcript,
3910                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3911                )
3912                .await?;
3913            compacted_this_turn = compacted_this_turn
3914                || transcript
3915                    .iter()
3916                    .any(crate::compaction::is_compaction_boundary);
3917
3918            let speed_policy_decision =
3919                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3920            let request_reasoning = reasoning_from_decision(
3921                speed_policy_decision.as_ref(),
3922                routing_selection
3923                    .reasoning
3924                    .clone()
3925                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3926            );
3927            self.active_turn_selections.write().await.insert(
3928                turn_id.clone(),
3929                ModelSelectionMode::manual(
3930                    provider.clone(),
3931                    model.clone(),
3932                    request_reasoning.level.clone(),
3933                ),
3934            );
3935            if let Some(limit) = reliability.record_model_call(
3936                &cfg.reliability,
3937                runtime_profile == RuntimeProfile::Interactive,
3938            ) {
3939                self.fail_turn_due_to_reliability_limit(
3940                    &req.thread_id,
3941                    &turn_id,
3942                    &provider,
3943                    &model,
3944                    limit,
3945                    &transcript,
3946                )
3947                .await?;
3948                return Ok(TurnRunOutcome::Stopped);
3949            }
3950            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3951                thread_id: req.thread_id.clone(),
3952                turn_id: turn_id.clone(),
3953                engine_id: engine.id(),
3954                model: ModelSelection {
3955                    provider: provider.clone(),
3956                    model: model.clone(),
3957                },
3958                reasoning: request_reasoning.clone(),
3959                speed_policy: speed_policy_decision.clone(),
3960                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3961                timestamp: OffsetDateTime::now_utc(),
3962            }))
3963            .await;
3964
3965            let mut instructions = req.instructions.clone();
3966            if let Some(extra) = &thread_overrides.developer_instructions {
3967                instructions = apply_thread_developer_instructions(instructions, extra);
3968            }
3969            if let Some(context) = req.developer_context.as_deref() {
3970                instructions = apply_turn_developer_context(instructions, context);
3971            }
3972            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3973            if let Some(profile) = &model_profile {
3974                instructions = apply_model_instruction_overlay(instructions, profile);
3975            }
3976            if req.task_ledger_required
3977                && runtime_profile == RuntimeProfile::Eval
3978                && !transcript_has_task_ledger(&transcript)
3979            {
3980                instructions = apply_task_ledger_required(instructions);
3981            }
3982            if effective_policy_mode == PolicyMode::Plan {
3983                instructions = apply_plan_mode(instructions);
3984            }
3985            if agent_swarm_mode_active {
3986                instructions = apply_agent_swarm_mode(instructions);
3987            }
3988            if model_supports_reasoning_effort(&model, REASONING_ULTRA) {
3989                instructions = apply_codex_multi_agent_mode(
3990                    instructions,
3991                    request_reasoning.level.as_deref() == Some(REASONING_ULTRA),
3992                );
3993            }
3994            instructions = self
3995                .goals
3996                .apply_goal_instructions(&req.thread_id, instructions)
3997                .await?;
3998            let mut request_metadata = serde_json::json!({});
3999            if let Some(decision) = &speed_policy_decision {
4000                request_metadata["speedPolicy"] = serde_json::json!(decision);
4001            }
4002            if let Some(decision) = routing_selection.decision.as_ref() {
4003                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4004            }
4005            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4006                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4007            }
4008            if let Some(profile) = &model_profile {
4009                request_metadata["modelProfile"] = serde_json::json!({
4010                    "model": profile.model,
4011                    "providerFamily": profile.provider_family,
4012                    "editTool": profile.edit_tool,
4013                    "schemaPolicy": profile.schema_policy,
4014                    "instructionOverlay": profile.instruction_overlay,
4015                    "parallelToolCalls": profile.parallel_tool_calls,
4016                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4017                });
4018            }
4019            let task_ledger_required_this_round = req.task_ledger_required
4020                && runtime_profile == RuntimeProfile::Eval
4021                && !deadline_finalization_requested
4022                && !transcript_has_task_ledger(&transcript);
4023            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4024                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4025                .filter(|tools| !tools.is_empty());
4026            let request_tools = if deadline_finalization_requested {
4027                Vec::new()
4028            } else if let Some(ledger_tools) = &task_ledger_tools {
4029                ledger_tools.clone()
4030            } else {
4031                tools.clone()
4032            };
4033            let request_tool_choice = if deadline_finalization_requested {
4034                ToolChoice::None
4035            } else if task_ledger_tools.is_some() {
4036                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4037            } else {
4038                tool_choice.clone()
4039            };
4040            if deadline_finalization_requested {
4041                request_metadata["deadlineFinalization"] = serde_json::json!({
4042                    "reserveSeconds": deadline_finalization_reserve,
4043                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4044                });
4045            } else if deadline_scoreable_completion_requested {
4046                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4047                    "reserveSeconds": deadline_finalization_reserve,
4048                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4049                });
4050            }
4051            let request = AgentInferenceRequest {
4052                model: ModelSelection {
4053                    provider: provider.clone(),
4054                    model: model.clone(),
4055                },
4056                instructions,
4057                transcript: transcript.clone(),
4058                tools: request_tools,
4059                tool_choice: request_tool_choice,
4060                reasoning: request_reasoning,
4061                output: OutputConfig::default(),
4062                runtime: RuntimeHints {
4063                    auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4064                    profile: runtime_profile,
4065                    parallel_tool_calls: Some(parallel_tool_calls),
4066                    hosted_web_search: cfg.hosted_web_search.clone(),
4067                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4068                    speed_policy: speed_policy_decision,
4069                    reliability: Some(cfg.reliability.clone().into()),
4070                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4071                    ..RuntimeHints::default()
4072                },
4073                metadata: request_metadata,
4074            };
4075
4076            let ctx = InferenceTurnContext {
4077                thread_id: &req.thread_id,
4078                turn_id: &turn_id,
4079                tool_executor: Some(std::sync::Arc::new(
4080                    crate::tool_execution::RuntimeTurnToolExecutor {
4081                        runtime: Arc::clone(self),
4082                        thread_id: req.thread_id.clone(),
4083                        turn_id: turn_id.clone(),
4084                        workspace: Some(workspace.clone()),
4085                        deadline: turn_deadline,
4086                    },
4087                )),
4088            };
4089            let stream_future = engine.stream_turn(ctx, request);
4090            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4091                turn_deadline,
4092                runtime_profile,
4093                req.task_ledger_required,
4094                deadline_finalization_reserve,
4095                deadline_finalization_requested || deadline_scoreable_completion_requested,
4096                task_ledger_scoreable_checkpoints,
4097                &transcript,
4098            ) {
4099                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4100                    Ok(stream) => stream?,
4101                    Err(_) => {
4102                        if runtime_profile == RuntimeProfile::Eval
4103                            && !deadline_finalization_requested
4104                        {
4105                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4106                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4107                                && task_ledger_scoreable_checkpoints
4108                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4109                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4110                            {
4111                                task_ledger_scoreable_checkpoints += 1;
4112                                let item = TranscriptItem::UserMessage(UserMessage::text(
4113                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4114                                ));
4115                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4116                                    .await?;
4117                                transcript.push(item);
4118                                continue 'tool_rounds;
4119                            }
4120                            self.start_deadline_finalization(
4121                                &req.thread_id,
4122                                &turn_id,
4123                                &mut transcript,
4124                                remaining,
4125                            )
4126                            .await?;
4127                            deadline_finalization_requested = true;
4128                            continue 'tool_rounds;
4129                        }
4130                        self.fail_turn_due_to_deadline(
4131                            &req.thread_id,
4132                            &turn_id,
4133                            deadline,
4134                            &transcript,
4135                        )
4136                        .await?;
4137                        return Ok(TurnRunOutcome::Stopped);
4138                    }
4139                }
4140            } else {
4141                stream_future.await?
4142            };
4143            let mut assistant_text = String::new();
4144            let mut phase_messages = Vec::<AssistantMessage>::new();
4145            let mut reasoning_text = String::new();
4146            let mut tool_calls = Vec::new();
4147            let mut provider_metadata = None;
4148            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4149            // sometimes abort the SSE stream after emitting the compaction item
4150            // ("error decoding response body") before response.completed, so we
4151            // must not rely solely on ProviderMetadata for the boundary.
4152            let mut stream_compaction_item: Option<serde_json::Value> = None;
4153
4154            loop {
4155                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4156                    turn_deadline,
4157                    runtime_profile,
4158                    req.task_ledger_required,
4159                    deadline_finalization_reserve,
4160                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4161                    task_ledger_scoreable_checkpoints,
4162                    &transcript,
4163                ) {
4164                    match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4165                        Ok(next) => next,
4166                        Err(_) => {
4167                            if runtime_profile == RuntimeProfile::Eval
4168                                && !deadline_finalization_requested
4169                            {
4170                                let remaining =
4171                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4172                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4173                                    && task_ledger_scoreable_checkpoints
4174                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4175                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4176                                {
4177                                    task_ledger_scoreable_checkpoints += 1;
4178                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4179                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4180                                    ));
4181                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4182                                        .await?;
4183                                    transcript.push(item);
4184                                    continue 'tool_rounds;
4185                                }
4186                                self.start_deadline_finalization(
4187                                    &req.thread_id,
4188                                    &turn_id,
4189                                    &mut transcript,
4190                                    remaining,
4191                                )
4192                                .await?;
4193                                deadline_finalization_requested = true;
4194                                continue 'tool_rounds;
4195                            }
4196                            self.fail_turn_due_to_deadline(
4197                                &req.thread_id,
4198                                &turn_id,
4199                                deadline,
4200                                &transcript,
4201                            )
4202                            .await?;
4203                            return Ok(TurnRunOutcome::Stopped);
4204                        }
4205                    }
4206                } else {
4207                    stream.next().await
4208                };
4209                let Some(res) = next else {
4210                    break;
4211                };
4212                let event = match res {
4213                    Ok(event) => event,
4214                    Err(err) => {
4215                        let error = err.to_string();
4216                        if runtime_profile == RuntimeProfile::Eval
4217                            && !deadline_finalization_requested
4218                            && let Some(cause) = provider_stream_retry_cause(&error)
4219                        {
4220                            let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4221                            let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4222                            if retry_attempt < policy.provider_retry_max_attempts {
4223                                provider_stream_retry_attempts = retry_attempt;
4224                                let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4225                                self.emit(RoderEvent::ReliabilityRetryRecorded(
4226                                    ReliabilityRetryRecorded {
4227                                        context: ReliabilityContext {
4228                                            thread_id: req.thread_id.clone(),
4229                                            turn_id: turn_id.clone(),
4230                                            provider: Some(provider.clone()),
4231                                            model: Some(model.clone()),
4232                                            ..ReliabilityContext::default()
4233                                        },
4234                                        error_class: ReliabilityErrorClass::ProviderError,
4235                                        decision: ReliabilityRetryDecision::Retry,
4236                                        attempt: retry_attempt,
4237                                        max_attempts: policy.provider_retry_max_attempts,
4238                                        delay_ms: Some(delay_ms),
4239                                        details: ReliabilityDetails::redacted(format!(
4240                                            "{cause}: {error}"
4241                                        )),
4242                                        timestamp: OffsetDateTime::now_utc(),
4243                                    },
4244                                ))
4245                                .await;
4246                                if delay_ms > 0 {
4247                                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4248                                        .await;
4249                                }
4250                                continue 'tool_rounds;
4251                            }
4252                        }
4253                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4254                            thread_id: req.thread_id.clone(),
4255                            turn_id: turn_id.clone(),
4256                            error: error.clone(),
4257                            error_kind: None,
4258                            usage: None,
4259                            timestamp: OffsetDateTime::now_utc(),
4260                        }))
4261                        .await;
4262                        self.complete_team_member_turn_with_result(
4263                            &req.thread_id,
4264                            &turn_id,
4265                            TeamMemberStatus::Failed,
4266                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4267                            Some(error),
4268                        )
4269                        .await?;
4270                        return Err(err);
4271                    }
4272                };
4273
4274                let inference_timestamp = OffsetDateTime::now_utc();
4275                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4276                    thread_id: req.thread_id.clone(),
4277                    turn_id: turn_id.clone(),
4278                    event: event.clone(),
4279                    timestamp: inference_timestamp,
4280                }))
4281                .await;
4282
4283                match event {
4284                    InferenceEvent::MessageDelta(delta) => {
4285                        if let Some((team_id, member)) =
4286                            self.teams.member_for_thread(&req.thread_id).await
4287                        {
4288                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4289                                team_id,
4290                                member_id: member.id,
4291                                member_thread_id: req.thread_id.clone(),
4292                                turn_id: turn_id.clone(),
4293                                delta: delta.text.clone(),
4294                                timestamp: OffsetDateTime::now_utc(),
4295                            }))
4296                            .await;
4297                        }
4298                        if is_final_answer_phase(delta.phase.as_deref()) {
4299                            assistant_text.push_str(&delta.text);
4300                        } else if let Some(last) = phase_messages.last_mut()
4301                            && last.phase == delta.phase
4302                        {
4303                            last.text.push_str(&delta.text);
4304                        } else {
4305                            phase_messages.push(AssistantMessage {
4306                                text: delta.text,
4307                                phase: delta.phase,
4308                            });
4309                        }
4310                    }
4311                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4312                    InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4313                    InferenceEvent::Failed(failure) => {
4314                        speed_policy.record_failure();
4315                        let error = failure.message;
4316                        self.persist_turn_item(
4317                            &req.thread_id,
4318                            &turn_id,
4319                            &TranscriptItem::Error(ErrorRecord {
4320                                message: error.clone(),
4321                            }),
4322                        )
4323                        .await?;
4324                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4325                            thread_id: req.thread_id.clone(),
4326                            turn_id: turn_id.clone(),
4327                            error: error.clone(),
4328                            error_kind: None,
4329                            usage: None,
4330                            timestamp: OffsetDateTime::now_utc(),
4331                        }))
4332                        .await;
4333                        self.complete_team_member_turn_with_result(
4334                            &req.thread_id,
4335                            &turn_id,
4336                            TeamMemberStatus::Failed,
4337                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4338                            Some(error),
4339                        )
4340                        .await?;
4341                        return Ok(TurnRunOutcome::Stopped);
4342                    }
4343                    InferenceEvent::Usage(usage) => {
4344                        turn_usage.add_assign(&usage);
4345                    }
4346                    InferenceEvent::Completed(metadata) => {
4347                        turn_finish_reason = metadata
4348                            .stop_reason
4349                            .as_deref()
4350                            .map(finish_reason_from_stop_reason);
4351                    }
4352                    InferenceEvent::Compaction(progress) => {
4353                        // Persist the boundary as soon as the provider emits a
4354                        // completed compaction item. ChatGPT Codex often aborts
4355                        // the SSE stream right after ("error decoding response
4356                        // body"), so waiting for ProviderMetadata loses the
4357                        // boundary and every subsequent request re-compacts.
4358                        if progress.status == "completed"
4359                            && let Some(item) = progress.item
4360                        {
4361                            let already = stream_compaction_item
4362                                .as_ref()
4363                                .and_then(|existing| {
4364                                    existing.get("id").and_then(serde_json::Value::as_str)
4365                                })
4366                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4367                                .is_some_and(|(a, b)| a == b);
4368                            if !already {
4369                                stream_compaction_item = Some(item.clone());
4370                                let boundary = TranscriptItem::ProviderMetadata(
4371                                    crate::compaction::provider_metadata_from_compaction_item(item),
4372                                );
4373                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4374                                    .await?;
4375                                transcript.push(boundary);
4376                                transcript =
4377                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4378                                compacted_this_turn = true;
4379                            }
4380                        }
4381                    }
4382                    InferenceEvent::HostedToolCallStarted(_)
4383                    | InferenceEvent::HostedToolCallCompleted(_)
4384                    | InferenceEvent::ToolCallStarted(_)
4385                    | InferenceEvent::ToolCallDelta(_) => {}
4386                    InferenceEvent::ProviderMetadata(mut metadata) => {
4387                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4388                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4389                                &mut metadata,
4390                                compaction_item,
4391                            );
4392                        }
4393                        provider_metadata = Some(metadata);
4394                    }
4395                }
4396            }
4397
4398            speed_policy.record_model_output(
4399                !assistant_text.is_empty() || !phase_messages.is_empty(),
4400                tool_calls.len(),
4401            );
4402            if tool_calls.is_empty() {
4403                let steers = self.drain_turn_steers(&turn_id).await;
4404                if !steers.is_empty() {
4405                    for message in phase_messages {
4406                        let item = TranscriptItem::AssistantMessage(message);
4407                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4408                            .await?;
4409                        transcript.push(item);
4410                        self.persist_model_profile_segment(
4411                            &req.thread_id,
4412                            &turn_id,
4413                            model_profile.as_ref(),
4414                            &provider,
4415                            &model,
4416                            "assistant",
4417                        )
4418                        .await?;
4419                    }
4420                    if !assistant_text.is_empty() {
4421                        let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4422                            text: assistant_text,
4423                            phase: Some(FINAL_ANSWER_PHASE.to_string()),
4424                        });
4425                        self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4426                            .await?;
4427                        transcript.push(assistant);
4428                        self.persist_model_profile_segment(
4429                            &req.thread_id,
4430                            &turn_id,
4431                            model_profile.as_ref(),
4432                            &provider,
4433                            &model,
4434                            "assistant",
4435                        )
4436                        .await?;
4437                    }
4438                    if let Some(metadata) = provider_metadata {
4439                        let had_provider_compaction =
4440                            crate::compaction::provider_metadata_has_compaction(&metadata);
4441                        let item = TranscriptItem::ProviderMetadata(metadata);
4442                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4443                            .await?;
4444                        transcript.push(item);
4445                        if had_provider_compaction {
4446                            // Server-side compaction replaced the prior window;
4447                            // drop pre-boundary items so the next request does
4448                            // not re-send (and re-compact) the full history.
4449                            transcript =
4450                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4451                            compacted_this_turn = true;
4452                        }
4453                    }
4454                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4455                        .await?;
4456                    continue;
4457                }
4458                if !deadline_finalization_requested
4459                    && req.task_ledger_required
4460                    && runtime_profile == RuntimeProfile::Eval
4461                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4462                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4463                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4464                {
4465                    task_ledger_completion_reminders += 1;
4466                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4467                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4468                        .await?;
4469                    transcript.push(item);
4470                    continue;
4471                }
4472                if !deadline_finalization_requested
4473                    && let Some(prompt) = verification_gate.blocking_prompt()
4474                {
4475                    speed_policy.record_verification_required();
4476                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4477                        thread_id: req.thread_id.clone(),
4478                        turn_id: turn_id.clone(),
4479                        reason: verification_gate.reason(),
4480                        changed_files: verification_gate.changed_files(),
4481                        tool_evidence: verification_gate.tool_evidence.clone(),
4482                        tests_run: verification_gate.tests_run.clone(),
4483                        open_gaps: verification_gate.open_gaps.clone(),
4484                        timestamp: OffsetDateTime::now_utc(),
4485                    }))
4486                    .await;
4487                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4488                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4489                        .await?;
4490                    transcript.push(item);
4491                    continue;
4492                }
4493                // Loop persistence nudge: in non-interactive/eval runs, when the
4494                // model returns a final message with no tool calls, gently prompt
4495                // it to keep going (bounded by `empty_tool_call_nudges`) before
4496                // ending the turn. Gated to non-interactive/eval so interactive
4497                // chat completions are never delayed, and only fires when the
4498                // model actually produced a final answer to re-examine.
4499                if !deadline_finalization_requested
4500                    && runtime_profile != RuntimeProfile::Interactive
4501                    && cfg.reliability.empty_tool_call_nudges > 0
4502                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4503                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4504                {
4505                    empty_tool_call_nudges_used += 1;
4506                    let item = TranscriptItem::UserMessage(UserMessage::text(
4507                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4508                    ));
4509                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4510                        .await?;
4511                    transcript.push(item);
4512                    continue;
4513                }
4514                if deadline_finalization_requested
4515                    && assistant_text.trim().is_empty()
4516                    && phase_messages.is_empty()
4517                {
4518                    assistant_text = format!(
4519                        "Deadline finalization completed without model text. {}",
4520                        turn_partial_result(&transcript)
4521                    );
4522                }
4523                final_phase_messages = phase_messages;
4524                final_assistant_text = assistant_text;
4525                final_reasoning_text = reasoning_text;
4526                final_provider_metadata = provider_metadata;
4527                exhausted_tool_rounds = false;
4528                break;
4529            }
4530
4531            for message in phase_messages {
4532                let item = TranscriptItem::AssistantMessage(message);
4533                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4534                    .await?;
4535                transcript.push(item);
4536                self.persist_model_profile_segment(
4537                    &req.thread_id,
4538                    &turn_id,
4539                    model_profile.as_ref(),
4540                    &provider,
4541                    &model,
4542                    "assistant",
4543                )
4544                .await?;
4545            }
4546            if !assistant_text.is_empty() {
4547                transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4548                    text: assistant_text,
4549                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4550                }));
4551                self.persist_model_profile_segment(
4552                    &req.thread_id,
4553                    &turn_id,
4554                    model_profile.as_ref(),
4555                    &provider,
4556                    &model,
4557                    "assistant",
4558                )
4559                .await?;
4560            }
4561            if let Some(metadata) = provider_metadata {
4562                let had_provider_compaction =
4563                    crate::compaction::provider_metadata_has_compaction(&metadata);
4564                let item = TranscriptItem::ProviderMetadata(metadata);
4565                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4566                    .await?;
4567                transcript.push(item);
4568                if had_provider_compaction {
4569                    // Server-side compaction replaced the prior window; drop
4570                    // pre-boundary items so subsequent tool rounds use the
4571                    // compact window as the next input.
4572                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4573                    compacted_this_turn = true;
4574                }
4575            }
4576            for call in &tool_calls {
4577                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4578                    id: call.id.clone(),
4579                    name: call.name.clone(),
4580                    arguments: call.arguments.clone(),
4581                });
4582                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4583                    .await?;
4584                transcript.push(tool_item);
4585                self.persist_model_profile_segment(
4586                    &req.thread_id,
4587                    &turn_id,
4588                    model_profile.as_ref(),
4589                    &provider,
4590                    &model,
4591                    "tool_call",
4592                )
4593                .await?;
4594            }
4595            if let Some(deadline) = turn_deadline
4596                && deadline_expired(deadline)
4597            {
4598                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4599                    .await?;
4600                return Ok(TurnRunOutcome::Stopped);
4601            }
4602            let results = self
4603                .route_tool_calls(
4604                    &req.thread_id,
4605                    &turn_id,
4606                    tool_calls,
4607                    parallel_tool_calls,
4608                    Some(workspace.as_str()),
4609                    turn_deadline,
4610                )
4611                .await?;
4612            let reliability_limit = reliability.record_tool_results(
4613                &cfg.reliability,
4614                &results,
4615                runtime_profile == RuntimeProfile::Interactive,
4616            );
4617            for result in results {
4618                verification_gate.record_tool_result(&result);
4619                transcript.push(TranscriptItem::ToolResult(result));
4620                self.persist_model_profile_segment(
4621                    &req.thread_id,
4622                    &turn_id,
4623                    model_profile.as_ref(),
4624                    &provider,
4625                    &model,
4626                    "tool_result",
4627                )
4628                .await?;
4629            }
4630            if let Some(limit) = reliability_limit {
4631                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4632                    // Loop persistence: rather than ending the turn, reset the
4633                    // consecutive-failure counter, nudge the model to keep going,
4634                    // and continue the round loop. Bounded by the per-turn tool
4635                    // failure ceiling, `max_model_calls_per_turn`, and
4636                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4637                    self.record_reliability_limit_continuation(
4638                        &req.thread_id,
4639                        &turn_id,
4640                        &provider,
4641                        &model,
4642                        limit,
4643                    )
4644                    .await;
4645                    reliability.reset_consecutive_failures();
4646                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4647                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4648                    ));
4649                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4650                        .await?;
4651                    transcript.push(nudge);
4652                } else {
4653                    self.fail_turn_due_to_reliability_limit(
4654                        &req.thread_id,
4655                        &turn_id,
4656                        &provider,
4657                        &model,
4658                        limit,
4659                        &transcript,
4660                    )
4661                    .await?;
4662                    return Ok(TurnRunOutcome::Stopped);
4663                }
4664            }
4665            transcript = self
4666                .compact_transcript_if_needed(
4667                    &req.thread_id,
4668                    &turn_id,
4669                    &provider,
4670                    &model,
4671                    transcript,
4672                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4673                )
4674                .await?;
4675            compacted_this_turn = compacted_this_turn
4676                || transcript
4677                    .iter()
4678                    .any(crate::compaction::is_compaction_boundary);
4679        }
4680
4681        if exhausted_tool_rounds {
4682            let message =
4683                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4684            self.persist_turn_item(
4685                &req.thread_id,
4686                &turn_id,
4687                &TranscriptItem::Error(ErrorRecord {
4688                    message: message.clone(),
4689                }),
4690            )
4691            .await?;
4692            self.emit(RoderEvent::TurnFailed(TurnFailed {
4693                thread_id: req.thread_id.clone(),
4694                turn_id: turn_id.clone(),
4695                error: message.clone(),
4696                error_kind: None,
4697                usage: None,
4698                timestamp: OffsetDateTime::now_utc(),
4699            }))
4700            .await;
4701            self.complete_team_member_turn_with_result(
4702                &req.thread_id,
4703                &turn_id,
4704                TeamMemberStatus::Failed,
4705                None,
4706                Some(message),
4707            )
4708            .await?;
4709            return Ok(TurnRunOutcome::Stopped);
4710        }
4711
4712        if !final_reasoning_text.is_empty() {
4713            self.persist_turn_item(
4714                &req.thread_id,
4715                &turn_id,
4716                &TranscriptItem::ReasoningSummary(ReasoningSummary {
4717                    text: final_reasoning_text,
4718                }),
4719            )
4720            .await?;
4721        }
4722        for message in final_phase_messages {
4723            self.persist_turn_item(
4724                &req.thread_id,
4725                &turn_id,
4726                &TranscriptItem::AssistantMessage(message),
4727            )
4728            .await?;
4729            self.persist_model_profile_segment(
4730                &req.thread_id,
4731                &turn_id,
4732                model_profile.as_ref(),
4733                &provider,
4734                &model,
4735                "assistant",
4736            )
4737            .await?;
4738        }
4739        if !final_assistant_text.is_empty() {
4740            self.persist_turn_item(
4741                &req.thread_id,
4742                &turn_id,
4743                &TranscriptItem::AssistantMessage(AssistantMessage {
4744                    text: final_assistant_text.clone(),
4745                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4746                }),
4747            )
4748            .await?;
4749            self.persist_model_profile_segment(
4750                &req.thread_id,
4751                &turn_id,
4752                model_profile.as_ref(),
4753                &provider,
4754                &model,
4755                "assistant",
4756            )
4757            .await?;
4758        }
4759        if let Some(metadata) = final_provider_metadata {
4760            self.persist_turn_item(
4761                &req.thread_id,
4762                &turn_id,
4763                &TranscriptItem::ProviderMetadata(metadata),
4764            )
4765            .await?;
4766        }
4767
4768        let turn_usage_tokens = turn_usage.total_tokens as i64;
4769        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4770        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4771            .await?;
4772        self.goals
4773            .account_turn_usage(
4774                &req.thread_id,
4775                turn_usage_tokens,
4776                OffsetDateTime::now_utc() - turn_started_at,
4777            )
4778            .await?;
4779        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4780        self.record_turn_lifecycle_with_ownership(
4781            req.thread_id.clone(),
4782            turn_id.clone(),
4783            TurnLifecycleState::Completed,
4784            cleanup,
4785            None,
4786            ownership,
4787        )
4788        .await;
4789        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4790            thread_id: req.thread_id.clone(),
4791            turn_id: turn_id.clone(),
4792            usage: completed_usage,
4793            finish_reason: turn_finish_reason,
4794            timestamp: OffsetDateTime::now_utc(),
4795        }))
4796        .await;
4797        self.complete_team_member_turn_with_result(
4798            &req.thread_id,
4799            &turn_id,
4800            TeamMemberStatus::Completed,
4801            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4802            None,
4803        )
4804        .await?;
4805        Ok(TurnRunOutcome::Completed)
4806    }
4807
4808    async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
4809        let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
4810            return Vec::new();
4811        };
4812        let mut steers = active.steers.lock().await;
4813        std::mem::take(&mut *steers)
4814    }
4815
4816    async fn route_tool_calls(
4817        self: &Arc<Self>,
4818        thread_id: &ThreadId,
4819        turn_id: &TurnId,
4820        calls: Vec<ToolCallCompleted>,
4821        parallel: bool,
4822        workspace: Option<&str>,
4823        deadline: Option<OffsetDateTime>,
4824    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4825        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4826        // `agent_swarm` must be the only tool call in a model response. A mixed
4827        // or multi-swarm batch is denied wholesale with actionable retry text so
4828        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4829        // still gets a response (keeping the chat-completions transcript valid).
4830        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4831            calls.iter().map(|call| call.name.as_str()),
4832        ) {
4833            let message = violation.deny_message();
4834            return Ok(calls
4835                .into_iter()
4836                .map(|call| ToolResultRecord {
4837                    id: call.id,
4838                    name: Some(call.name),
4839                    result: message.clone(),
4840                    display_payload: None,
4841                    is_error: true,
4842                })
4843                .collect());
4844        }
4845        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4846        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4847        // progress flows through the existing Subagent* trace events.
4848        let swarm_call = (calls.len() == 1
4849            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4850            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4851        if let Some((tool_id, args)) = &swarm_call {
4852            self.emit(RoderEvent::AgentSwarmStarted(
4853                roder_api::subagents::AgentSwarmStarted {
4854                    thread_id: thread_id.clone(),
4855                    turn_id: turn_id.clone(),
4856                    tool_id: tool_id.clone(),
4857                    child_count: agent_swarm_child_count(args),
4858                    timestamp: OffsetDateTime::now_utc(),
4859                },
4860            ))
4861            .await;
4862        }
4863
4864        let force_sequential = calls
4865            .iter()
4866            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4867        let results =
4868            if parallel && !force_sequential {
4869                try_join_all(calls.into_iter().map(|call| {
4870                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4871                }))
4872                .await
4873            } else {
4874                let mut results = Vec::with_capacity(calls.len());
4875                for call in calls {
4876                    results.push(
4877                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4878                            .await?,
4879                    );
4880                }
4881                Ok(results)
4882            }?;
4883
4884        if let Some((tool_id, _)) = &swarm_call
4885            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4886            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4887        {
4888            self.emit(RoderEvent::AgentSwarmCompleted(
4889                roder_api::subagents::AgentSwarmCompleted {
4890                    thread_id: thread_id.clone(),
4891                    turn_id: turn_id.clone(),
4892                    tool_id: tool_id.clone(),
4893                    completed,
4894                    failed,
4895                    aborted,
4896                    timestamp: OffsetDateTime::now_utc(),
4897                },
4898            ))
4899            .await;
4900        }
4901
4902        Ok(results)
4903    }
4904
4905    async fn fail_turn_with_error(
4906        &self,
4907        thread_id: &ThreadId,
4908        turn_id: &TurnId,
4909        message: String,
4910    ) -> anyhow::Result<()> {
4911        self.persist_turn_item(
4912            thread_id,
4913            turn_id,
4914            &TranscriptItem::Error(ErrorRecord {
4915                message: message.clone(),
4916            }),
4917        )
4918        .await?;
4919        self.emit(RoderEvent::TurnFailed(TurnFailed {
4920            thread_id: thread_id.clone(),
4921            turn_id: turn_id.clone(),
4922            error: message.clone(),
4923            error_kind: None,
4924            usage: None,
4925            timestamp: OffsetDateTime::now_utc(),
4926        }))
4927        .await;
4928        self.complete_team_member_turn_with_result(
4929            thread_id,
4930            turn_id,
4931            TeamMemberStatus::Failed,
4932            None,
4933            Some(message),
4934        )
4935        .await?;
4936        Ok(())
4937    }
4938
4939    async fn fail_turn_due_to_deadline(
4940        &self,
4941        thread_id: &ThreadId,
4942        turn_id: &TurnId,
4943        deadline: OffsetDateTime,
4944        transcript: &[TranscriptItem],
4945    ) -> anyhow::Result<()> {
4946        let partial_result = turn_partial_result(transcript);
4947        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
4948            thread_id: thread_id.clone(),
4949            turn_id: turn_id.clone(),
4950            summary: partial_result.clone(),
4951            timestamp: OffsetDateTime::now_utc(),
4952        }))
4953        .await;
4954        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
4955            thread_id: thread_id.clone(),
4956            turn_id: turn_id.clone(),
4957            deadline,
4958            partial_result: partial_result.clone(),
4959            timestamp: OffsetDateTime::now_utc(),
4960        }))
4961        .await;
4962        let message = "turn deadline expired".to_string();
4963        self.persist_turn_item(
4964            thread_id,
4965            turn_id,
4966            &TranscriptItem::Error(ErrorRecord {
4967                message: format!("{message}: {partial_result}"),
4968            }),
4969        )
4970        .await?;
4971        self.emit(RoderEvent::TurnFailed(TurnFailed {
4972            thread_id: thread_id.clone(),
4973            turn_id: turn_id.clone(),
4974            error: message.clone(),
4975            error_kind: Some("deadline_timeout".to_string()),
4976            usage: None,
4977            timestamp: OffsetDateTime::now_utc(),
4978        }))
4979        .await;
4980        self.complete_team_member_turn_with_result(
4981            thread_id,
4982            turn_id,
4983            TeamMemberStatus::Failed,
4984            None,
4985            Some(format!("{message}: {partial_result}")),
4986        )
4987        .await?;
4988        Ok(())
4989    }
4990
4991    async fn start_deadline_finalization(
4992        &self,
4993        thread_id: &ThreadId,
4994        turn_id: &TurnId,
4995        transcript: &mut Vec<TranscriptItem>,
4996        remaining_seconds: u64,
4997    ) -> anyhow::Result<()> {
4998        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
4999            remaining_seconds,
5000        ));
5001        self.persist_turn_item(thread_id, turn_id, &item).await?;
5002        transcript.push(item);
5003        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5004            thread_id: thread_id.clone(),
5005            turn_id: turn_id.clone(),
5006            summary: turn_partial_result(transcript),
5007            timestamp: OffsetDateTime::now_utc(),
5008        }))
5009        .await;
5010        Ok(())
5011    }
5012
5013    /// Emits the reliability-limit event for a continuation (loop persistence)
5014    /// without failing the turn. The caller resets the failure counter and
5015    /// injects a nudge so the round loop keeps going.
5016    async fn record_reliability_limit_continuation(
5017        &self,
5018        thread_id: &ThreadId,
5019        turn_id: &TurnId,
5020        provider: &str,
5021        model: &str,
5022        limit: ReliabilityLimitHit,
5023    ) {
5024        self.emit(RoderEvent::ReliabilityLimitRecorded(
5025            ReliabilityLimitRecorded {
5026                context: ReliabilityContext {
5027                    thread_id: thread_id.clone(),
5028                    turn_id: turn_id.clone(),
5029                    tool_id: None,
5030                    tool_name: None,
5031                    provider: Some(provider.to_string()),
5032                    model: Some(model.to_string()),
5033                },
5034                error_class: limit.error_class,
5035                limit_kind: limit.limit_kind,
5036                decision: limit.decision,
5037                current: limit.current,
5038                limit: limit.limit,
5039                details: ReliabilityDetails::redacted(&limit.message),
5040                timestamp: OffsetDateTime::now_utc(),
5041            },
5042        ))
5043        .await;
5044    }
5045
5046    async fn fail_turn_due_to_reliability_limit(
5047        &self,
5048        thread_id: &ThreadId,
5049        turn_id: &TurnId,
5050        provider: &str,
5051        model: &str,
5052        limit: ReliabilityLimitHit,
5053        transcript: &[TranscriptItem],
5054    ) -> anyhow::Result<()> {
5055        self.emit(RoderEvent::ReliabilityLimitRecorded(
5056            ReliabilityLimitRecorded {
5057                context: ReliabilityContext {
5058                    thread_id: thread_id.clone(),
5059                    turn_id: turn_id.clone(),
5060                    tool_id: None,
5061                    tool_name: None,
5062                    provider: Some(provider.to_string()),
5063                    model: Some(model.to_string()),
5064                },
5065                error_class: limit.error_class,
5066                limit_kind: limit.limit_kind,
5067                decision: limit.decision,
5068                current: limit.current,
5069                limit: limit.limit,
5070                details: ReliabilityDetails::redacted(&limit.message),
5071                timestamp: OffsetDateTime::now_utc(),
5072            },
5073        ))
5074        .await;
5075        let partial_result = turn_partial_result(transcript);
5076        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5077            thread_id: thread_id.clone(),
5078            turn_id: turn_id.clone(),
5079            summary: partial_result.clone(),
5080            timestamp: OffsetDateTime::now_utc(),
5081        }))
5082        .await;
5083        let message = format!("reliability limit reached: {}", limit.message);
5084        self.persist_turn_item(
5085            thread_id,
5086            turn_id,
5087            &TranscriptItem::Error(ErrorRecord {
5088                message: format!("{message}: {partial_result}"),
5089            }),
5090        )
5091        .await?;
5092        self.emit(RoderEvent::TurnFailed(TurnFailed {
5093            thread_id: thread_id.clone(),
5094            turn_id: turn_id.clone(),
5095            error: message.clone(),
5096            error_kind: Some("reliability_limit".to_string()),
5097            usage: None,
5098            timestamp: OffsetDateTime::now_utc(),
5099        }))
5100        .await;
5101        self.complete_team_member_turn_with_result(
5102            thread_id,
5103            turn_id,
5104            TeamMemberStatus::Failed,
5105            None,
5106            Some(format!("{message}: {partial_result}")),
5107        )
5108        .await?;
5109        Ok(())
5110    }
5111
5112    async fn append_steers(
5113        &self,
5114        req: &StartTurnRequest,
5115        turn_id: &TurnId,
5116        transcript: &mut Vec<TranscriptItem>,
5117        steers: Vec<QueuedTurnSteer>,
5118    ) -> anyhow::Result<()> {
5119        for queued in steers {
5120            let mut steer = queued.message;
5121            steer.text = steer.text.trim().to_string();
5122            if steer.text.is_empty() && steer.images.is_empty() {
5123                continue;
5124            }
5125            let item = TranscriptItem::UserMessage(steer);
5126            self.persist_turn_item(&req.thread_id, turn_id, &item)
5127                .await?;
5128            transcript.push(item);
5129            if let Some(ack) = queued.mailbox_ack {
5130                self.teams
5131                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5132                    .await?;
5133            }
5134        }
5135        Ok(())
5136    }
5137
5138    async fn persist_model_profile_segment(
5139        &self,
5140        thread_id: &ThreadId,
5141        turn_id: &TurnId,
5142        profile: Option<&ModelHarnessProfile>,
5143        provider: &str,
5144        model: &str,
5145        segment: &str,
5146    ) -> anyhow::Result<()> {
5147        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5148            profile, provider, model, segment,
5149        ));
5150        self.persist_turn_item(thread_id, turn_id, &item).await
5151    }
5152
5153    /**
5154     * Allowlists apply to built-in tools only; external tools are advertised with their
5155     * host-supplied schemas as given. An external tool shadows a built-in with the same name in
5156     * both advertisement and dispatch (see `route_tool_call`).
5157     */
5158    fn filtered_tool_specs(
5159        &self,
5160        cfg: &RuntimeConfig,
5161        model: &str,
5162        profile: Option<&ModelHarnessProfile>,
5163        thread_allowlist: &[String],
5164        external_tools: &[roder_api::tools::ToolSpec],
5165    ) -> Vec<roder_api::tools::ToolSpec> {
5166        let mut specs = self
5167            .tool_registry
5168            .specs_for_edit_tool_with_schema_policy(
5169                edit_tool_for_model(cfg, model),
5170                schema_policy_for_model(profile),
5171            )
5172            .into_iter()
5173            .filter(|spec| {
5174                allowlist_permits(&cfg.tool_allowlist, &spec.name)
5175                    && allowlist_permits(thread_allowlist, &spec.name)
5176                    && !external_tools.iter().any(|tool| tool.name == spec.name)
5177            })
5178            .collect::<Vec<_>>();
5179        specs.extend(external_tools.iter().cloned());
5180        specs
5181    }
5182
5183    fn task_ledger_tool_specs(
5184        &self,
5185        profile: Option<&ModelHarnessProfile>,
5186    ) -> Vec<roder_api::tools::ToolSpec> {
5187        self.tool_registry
5188            .get(TASK_LEDGER_TOOL_NAME)
5189            .map(|tool| {
5190                tool.spec()
5191                    .normalized_for_model_profile(schema_policy_for_model(profile))
5192            })
5193            .into_iter()
5194            .collect()
5195    }
5196
5197    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5198        self.registry
5199            .inference_engine(provider)
5200            .or_else(|| {
5201                self.registry
5202                    .default_inference_engine()
5203                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5204            })
5205            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5206    }
5207
5208    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5209        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5210            let _ = self.persist_turn_lifecycle_record(&record).await;
5211        }
5212        let envelope = self.bus.emit(event);
5213        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5214            && should_persist_thread_event(thread_id)
5215        {
5216            let _ = store.append_event(thread_id, &envelope).await;
5217        }
5218        // Registered event sinks (e.g. process extensions) receive the
5219        // persisted envelope through bounded per-sink queues; a slow sink
5220        // never blocks emit or turn progress.
5221        let dispatcher = self
5222            .event_sink_dispatcher
5223            .get_or_init(|| async {
5224                crate::event_sink_dispatch::EventSinkDispatcher::start(
5225                    &self.registry.event_sinks,
5226                    self.bus.clone(),
5227                )
5228            })
5229            .await;
5230        if !dispatcher.is_empty() {
5231            dispatcher.dispatch(&envelope, &self.bus);
5232        }
5233        envelope
5234    }
5235
5236    /// Records a projected thread item event and persists it through the configured thread store.
5237    ///
5238    /// App-server protocol notification bridges currently call this after translating runtime
5239    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5240    /// must make the same call if they need the derived item event stream persisted.
5241    pub async fn record_thread_item_event_kind(
5242        &self,
5243        thread_id: &ThreadId,
5244        turn_id: &TurnId,
5245        timestamp: OffsetDateTime,
5246        kind: ThreadItemEventKind,
5247    ) -> anyhow::Result<ThreadItemEvent> {
5248        let seq = self.next_thread_item_event_seq(thread_id).await?;
5249        let item_event = ThreadItemEvent {
5250            seq,
5251            event_id: format!("{turn_id}-item-event-{seq}"),
5252            thread_id: thread_id.clone(),
5253            turn_id: turn_id.clone(),
5254            timestamp,
5255            event: kind,
5256        };
5257        if let Some(store) = &self.thread_store {
5258            store.append_item_event(thread_id, &item_event).await?;
5259        }
5260        self.remember_thread_item_event(&item_event).await?;
5261        Ok(item_event)
5262    }
5263
5264    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5265        self.ensure_thread_item_cache(thread_id).await?;
5266        Ok(self
5267            .thread_item_cache
5268            .lock()
5269            .await
5270            .next_item_event_seq(thread_id))
5271    }
5272
5273    pub async fn thread_item_exists(
5274        &self,
5275        thread_id: &ThreadId,
5276        turn_id: &TurnId,
5277        item_id: &str,
5278    ) -> anyhow::Result<bool> {
5279        self.ensure_thread_item_cache(thread_id).await?;
5280        Ok(self
5281            .thread_item_cache
5282            .lock()
5283            .await
5284            .thread_item_exists(thread_id, turn_id, item_id))
5285    }
5286
5287    pub async fn current_reasoning_item_id(
5288        &self,
5289        thread_id: &ThreadId,
5290        turn_id: &TurnId,
5291    ) -> anyhow::Result<Option<String>> {
5292        self.ensure_thread_item_cache(thread_id).await?;
5293        Ok(self
5294            .thread_item_cache
5295            .lock()
5296            .await
5297            .current_reasoning_item_id(thread_id, turn_id))
5298    }
5299
5300    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5301        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5302        self.thread_item_cache
5303            .lock()
5304            .await
5305            .remember_item_event(item_event);
5306        Ok(())
5307    }
5308
5309    pub async fn latest_transcript_item_index(
5310        &self,
5311        thread_id: &ThreadId,
5312        turn_id: &TurnId,
5313    ) -> anyhow::Result<Option<usize>> {
5314        self.ensure_thread_item_cache(thread_id).await?;
5315        Ok(self
5316            .thread_item_cache
5317            .lock()
5318            .await
5319            .latest_transcript_item_index(thread_id, turn_id))
5320    }
5321
5322    async fn next_transcript_item_index(
5323        &self,
5324        thread_id: &ThreadId,
5325        turn_id: &TurnId,
5326    ) -> anyhow::Result<usize> {
5327        self.ensure_thread_item_cache(thread_id).await?;
5328        Ok(self
5329            .thread_item_cache
5330            .lock()
5331            .await
5332            .next_transcript_item_index(thread_id, turn_id))
5333    }
5334
5335    async fn remember_transcript_item_index(
5336        &self,
5337        thread_id: &ThreadId,
5338        turn_id: &TurnId,
5339        item_index: usize,
5340    ) -> anyhow::Result<()> {
5341        self.ensure_thread_item_cache(thread_id).await?;
5342        self.thread_item_cache
5343            .lock()
5344            .await
5345            .remember_transcript_item_index(thread_id, turn_id, item_index);
5346        Ok(())
5347    }
5348
5349    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5350        if self
5351            .thread_item_cache
5352            .lock()
5353            .await
5354            .contains_thread(thread_id)
5355        {
5356            return Ok(());
5357        }
5358
5359        let snapshot = if let Some(store) = &self.thread_store {
5360            store.load_thread(thread_id).await?
5361        } else {
5362            None
5363        };
5364        self.thread_item_cache.lock().await.ensure_thread(
5365            thread_id,
5366            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5367        );
5368        Ok(())
5369    }
5370
5371    pub(crate) async fn persist_turn_item(
5372        &self,
5373        thread_id: &ThreadId,
5374        turn_id: &TurnId,
5375        item: &TranscriptItem,
5376    ) -> anyhow::Result<()> {
5377        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5378        let timestamp = OffsetDateTime::now_utc();
5379        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5380            thread_id: thread_id.clone(),
5381            turn_id: turn_id.clone(),
5382            item_type: match item {
5383                TranscriptItem::UserMessage(_) => "user_message",
5384                TranscriptItem::AssistantMessage(_) => "assistant_message",
5385                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5386                TranscriptItem::ToolCall(_) => "tool_call",
5387                TranscriptItem::ToolResult(_) => "tool_result",
5388                TranscriptItem::FileChange(_) => "file_change",
5389                TranscriptItem::ContextCompaction(_) => "context_compaction",
5390                TranscriptItem::Error(_) => "error",
5391                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5392            }
5393            .to_string(),
5394            item_index: Some(item_index),
5395            item: Some(item.clone()),
5396            timestamp,
5397        }))
5398        .await;
5399        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5400            .await?;
5401        Ok(())
5402    }
5403}
5404
5405fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5406    transcript.iter().any(|item| {
5407        matches!(
5408            item,
5409            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5410        )
5411    })
5412}
5413
5414fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5415    transcript.iter().any(|item| {
5416        matches!(
5417            item,
5418            TranscriptItem::ToolResult(result)
5419                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5420        )
5421    })
5422}
5423
5424fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5425    let latest = transcript.iter().rev().find_map(|item| match item {
5426        TranscriptItem::ToolResult(result)
5427            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5428        {
5429            Some(result.result.as_str())
5430        }
5431        _ => None,
5432    })?;
5433    if !task_ledger_has_open_items(latest) {
5434        return None;
5435    }
5436
5437    let mut ledger = latest.chars().take(1500).collect::<String>();
5438    if latest.chars().nth(1500).is_some() {
5439        ledger.push_str("...");
5440    }
5441    Some(format!(
5442        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5443    ))
5444}
5445
5446fn task_ledger_deadline_completion_prompt(
5447    remaining_seconds: u64,
5448    reserve_seconds: u64,
5449    completion_prompt: &str,
5450) -> String {
5451    format!(
5452        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5453    )
5454}
5455
5456fn task_ledger_scoreable_checkpoint_prompt(
5457    remaining_seconds: u64,
5458    completion_prompt: &str,
5459) -> String {
5460    format!(
5461        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5462    )
5463}
5464
5465fn task_ledger_has_open_items(ledger: &str) -> bool {
5466    ledger.lines().any(|line| {
5467        let line = line.trim_start();
5468        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5469    })
5470}
5471
5472fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5473    if !cfg.runtime_profile.is_non_interactive() {
5474        return None;
5475    }
5476    cfg.turn_deadline_seconds
5477        .filter(|seconds| *seconds > 0)
5478        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5479}
5480
5481fn deadline_expired(deadline: OffsetDateTime) -> bool {
5482    OffsetDateTime::now_utc() >= deadline
5483}
5484
5485pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5486    let deadline = deadline?;
5487    if deadline <= OffsetDateTime::now_utc() {
5488        return Some(0);
5489    }
5490    Some(
5491        (deadline - OffsetDateTime::now_utc())
5492            .unsigned_abs()
5493            .as_secs()
5494            .max(1),
5495    )
5496}
5497
5498fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5499    let now = OffsetDateTime::now_utc();
5500    if deadline <= now {
5501        return tokio::time::Instant::now();
5502    }
5503    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5504}
5505
5506fn inference_timeout_deadline(
5507    deadline: Option<OffsetDateTime>,
5508    runtime_profile: RuntimeProfile,
5509    task_ledger_required: bool,
5510    reserve_seconds: u64,
5511    finalization_requested: bool,
5512    task_ledger_scoreable_checkpoints: u8,
5513    transcript: &[TranscriptItem],
5514) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5515    let deadline = deadline?;
5516    if runtime_profile == RuntimeProfile::Eval
5517        && task_ledger_required
5518        && !finalization_requested
5519        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5520        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5521        && task_ledger_completion_prompt(transcript).is_some()
5522    {
5523        let checkpoint_deadline =
5524            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5525        if checkpoint_deadline > OffsetDateTime::now_utc() {
5526            return Some((
5527                checkpoint_deadline,
5528                InferenceTimeoutAction::ScoreableCheckpoint,
5529            ));
5530        }
5531    }
5532    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5533        return Some((
5534            deadline - Duration::seconds(reserve_seconds as i64),
5535            InferenceTimeoutAction::Finalization,
5536        ));
5537    }
5538    Some((deadline, InferenceTimeoutAction::Finalization))
5539}
5540
5541fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5542    let tool_results = transcript
5543        .iter()
5544        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5545        .count();
5546    let assistant_messages = transcript
5547        .iter()
5548        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5549        .count();
5550    format!(
5551        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5552        transcript.len()
5553    )
5554}
5555
5556fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5557    let level = effective_reasoning_for_model(cfg, model);
5558    match level.as_str() {
5559        "" | REASONING_NONE => ReasoningConfig::default(),
5560        level => ReasoningConfig {
5561            enabled: true,
5562            level: Some(level.to_string()),
5563        },
5564    }
5565}
5566
5567fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5568    let entry = lookup_model(model)?;
5569    if !entry.supports_compaction {
5570        return None;
5571    }
5572    cfg.auto_compact_token_limit
5573        .or_else(|| {
5574            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5575        })
5576        .or(Some(entry.auto_compact_token_limit))
5577        .filter(|threshold| *threshold > 0)
5578}
5579
5580pub(crate) fn tool_search_for_provider_model(
5581    cfg: &RuntimeConfig,
5582    provider: &str,
5583    model: &str,
5584) -> ToolSearchConfig {
5585    let mut resolved = cfg.tool_search.clone();
5586    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5587        provider_config.apply_to(&mut resolved);
5588    }
5589    if let Some(model_config) = cfg.model_tool_search.get(model) {
5590        model_config.apply_to(&mut resolved);
5591    }
5592    resolved
5593}
5594
5595fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5596    cfg.model_parallel_tool_calls
5597        .get(model)
5598        .copied()
5599        .or_else(|| {
5600            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5601        })
5602        .unwrap_or(true)
5603}
5604
5605/// Count the children an `agent_swarm` call will launch from its arguments
5606/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5607fn agent_swarm_child_count(arguments: &str) -> usize {
5608    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5609        .map(|request| request.items.len() + request.resume_agent_ids.len())
5610        .unwrap_or(0)
5611}
5612
5613/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5614/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5615/// buckets default to 0. Returns `None` when the text is not a swarm result.
5616fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5617    if !text.contains("<agent_swarm_result>") {
5618        return None;
5619    }
5620    let bucket = |label: &str| -> usize {
5621        let needle = format!("{label}: ");
5622        text.find(&needle)
5623            .map(|start| start + needle.len())
5624            .map(|start| {
5625                text[start..]
5626                    .chars()
5627                    .take_while(|c| c.is_ascii_digit())
5628                    .collect::<String>()
5629            })
5630            .and_then(|digits| digits.parse().ok())
5631            .unwrap_or(0)
5632    };
5633    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5634}
5635
5636fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5637    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5638    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5639        return ultracode_reasoning_level_for_model(
5640            model,
5641            &cfg.speed_policy.ultracode_reasoning,
5642            &base_reasoning,
5643        );
5644    }
5645    base_reasoning
5646}
5647
5648fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5649    let Some(entry) = lookup_model(model) else {
5650        return cfg
5651            .reasoning
5652            .clone()
5653            .unwrap_or_else(|| REASONING_NONE.to_string());
5654    };
5655    if entry.supported_reasoning.is_empty() {
5656        return REASONING_NONE.to_string();
5657    }
5658    cfg.reasoning
5659        .as_deref()
5660        .filter(|reasoning| {
5661            entry
5662                .supported_reasoning
5663                .iter()
5664                .any(|option| option.effort == *reasoning)
5665        })
5666        .map(str::to_string)
5667        .or_else(|| {
5668            model_profile_for_model(cfg, model)
5669                .and_then(|profile| profile.reasoning.orientation)
5670                .filter(|reasoning| {
5671                    entry
5672                        .supported_reasoning
5673                        .iter()
5674                        .any(|option| option.effort == reasoning)
5675                })
5676        })
5677        .unwrap_or_else(|| entry.default_reasoning.to_string())
5678}
5679
5680fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5681    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5682        return Ok(());
5683    }
5684    let Some(entry) = lookup_model(model) else {
5685        return Ok(());
5686    };
5687    if entry
5688        .supported_reasoning
5689        .iter()
5690        .any(|option| option.effort == effort)
5691    {
5692        Ok(())
5693    } else {
5694        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5695    }
5696}
5697
5698fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5699    let Some(reasoning) = cfg.reasoning.as_deref() else {
5700        return Ok(());
5701    };
5702    let Some(entry) = lookup_model(&cfg.default_model) else {
5703        return Ok(());
5704    };
5705    if entry.provider != PROVIDER_GEMINI {
5706        return Ok(());
5707    }
5708    validate_reasoning_effort(&cfg.default_model, reasoning)
5709}
5710
5711fn validate_runtime_inference_router_config(
5712    registry: &ExtensionRegistry,
5713    cfg: &RuntimeConfig,
5714) -> anyhow::Result<()> {
5715    if !cfg.inference_router.enabled {
5716        return Ok(());
5717    }
5718    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5719        anyhow::bail!("inference_router.enabled requires inference_router.router");
5720    };
5721    if registry.inference_router(router_id).is_some() {
5722        return Ok(());
5723    }
5724    let available = registry
5725        .inference_routers
5726        .iter()
5727        .map(|router| router.id())
5728        .collect::<Vec<_>>()
5729        .join(", ");
5730    if available.is_empty() {
5731        anyhow::bail!("inference router {router_id:?} is not registered");
5732    }
5733    anyhow::bail!(
5734        "inference router {router_id:?} is not registered; available routers: {available}"
5735    );
5736}
5737
5738fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5739    lookup_model(model)
5740        .map(|entry| {
5741            entry
5742                .supported_reasoning
5743                .iter()
5744                .any(|option| option.effort == effort)
5745        })
5746        .unwrap_or(false)
5747}
5748
5749fn is_final_answer_phase(phase: Option<&str>) -> bool {
5750    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5751}
5752
5753fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5754    cfg.model_edit_tools
5755        .get(model)
5756        .map(String::as_str)
5757        .or_else(|| {
5758            cfg.model_profiles
5759                .get(model)
5760                .and_then(|profile| profile.edit_tool.as_deref())
5761        })
5762        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5763        .or(Some(EDIT_TOOL_EDIT))
5764}
5765
5766fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5767    cfg.model_profiles
5768        .get(model)
5769        .cloned()
5770        .or_else(|| built_in_model_profile(model))
5771}
5772
5773pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5774    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5775}
5776
5777/// Provider-aware profile resolution for the active turn.
5778///
5779/// Many model ids are shared across providers (for example Cursor proxies
5780/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5781/// first catalog entry, which assigns cross-provider ids the wrong family and
5782/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5783/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5784/// catalog entry, keeping user-configured profile overrides as the top
5785/// precedence.
5786fn model_profile_for_provider_model(
5787    cfg: &RuntimeConfig,
5788    provider: &str,
5789    model: &str,
5790) -> Option<ModelHarnessProfile> {
5791    cfg.model_profiles
5792        .get(model)
5793        .cloned()
5794        .or_else(|| built_in_model_profile_for_provider(provider, model))
5795}
5796
5797fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5798    profile
5799        .map(|profile| profile.schema_policy)
5800        .unwrap_or_default()
5801}
5802
5803fn model_profile_segment_metadata(
5804    profile: Option<&ModelHarnessProfile>,
5805    provider: &str,
5806    model: &str,
5807    segment: &str,
5808) -> serde_json::Value {
5809    serde_json::json!({
5810        "kind": MODEL_PROFILE_TRACE_KIND,
5811        "segment": segment,
5812        "provider": provider,
5813        "model": model,
5814        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5815        "providerFamily": profile.map(|profile| profile.provider_family),
5816        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5817        "schemaPolicy": profile.map(|profile| profile.schema_policy),
5818        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5819        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5820        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5821    })
5822}
5823
5824fn model_switch_summary(
5825    transcript: &[TranscriptItem],
5826    profile: Option<&ModelHarnessProfile>,
5827    provider: &str,
5828    model: &str,
5829    tools: &[roder_api::tools::ToolSpec],
5830) -> Option<String> {
5831    let previous = latest_model_profile_segment(transcript)?;
5832    let previous_model = previous
5833        .get("model")
5834        .and_then(serde_json::Value::as_str)
5835        .unwrap_or_default();
5836    let previous_provider = previous
5837        .get("provider")
5838        .and_then(serde_json::Value::as_str)
5839        .unwrap_or_default();
5840    if previous_model == model && previous_provider == provider {
5841        return None;
5842    }
5843
5844    let previous_profile = previous
5845        .get("profileModel")
5846        .and_then(serde_json::Value::as_str)
5847        .unwrap_or(previous_model);
5848    let current_profile = profile
5849        .map(|profile| profile.model.as_str())
5850        .unwrap_or(model);
5851    let previous_edit_tool = previous
5852        .get("editTool")
5853        .and_then(serde_json::Value::as_str)
5854        .unwrap_or("none");
5855    let current_edit_tool = profile
5856        .and_then(|profile| profile.edit_tool.as_deref())
5857        .unwrap_or("none");
5858    let tool_names = tools
5859        .iter()
5860        .map(|tool| tool.name.as_str())
5861        .take(12)
5862        .collect::<Vec<_>>()
5863        .join(", ");
5864    Some(format!(
5865        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
5866        if tool_names.is_empty() {
5867            "none"
5868        } else {
5869            &tool_names
5870        }
5871    ))
5872}
5873
5874fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
5875    transcript.iter().rev().find_map(|item| {
5876        let TranscriptItem::ProviderMetadata(value) = item else {
5877            return None;
5878        };
5879        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
5880            .then_some(value)
5881    })
5882}
5883
5884pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
5885    match value.trim() {
5886        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
5887        _ => anyhow::bail!(
5888            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
5889        ),
5890    }
5891}
5892
5893fn should_persist_thread_event(thread_id: &str) -> bool {
5894    !is_synthetic_event_thread_id(thread_id)
5895}
5896
5897#[cfg(test)]
5898#[path = "runtime/codex_v2_tests.rs"]
5899mod codex_v2_tests;
5900
5901#[cfg(test)]
5902#[path = "runtime/codex_v2_lifecycle_tests.rs"]
5903mod codex_v2_lifecycle_tests;
5904
5905#[cfg(test)]
5906mod tests {
5907    use super::*;
5908    use futures::stream;
5909    use roder_api::catalog::{
5910        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
5911        REASONING_NONE, REASONING_XHIGH,
5912    };
5913    use roder_api::extension::ExtensionRegistryBuilder;
5914    use roder_api::inference::{
5915        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
5916        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
5917        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
5918        ReasoningEffortDescriptor,
5919    };
5920    use roder_api::inference_routing::{
5921        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
5922    };
5923    use roder_api::thread::ThreadStoreFactory;
5924    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
5925    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
5926    use std::sync::Mutex as StdMutex;
5927
5928    fn test_workspace() -> String {
5929        std::env::current_dir().unwrap().display().to_string()
5930    }
5931
5932    struct MetadataMissingStore;
5933
5934    #[async_trait::async_trait]
5935    impl ThreadStore for MetadataMissingStore {
5936        fn id(&self) -> roder_api::thread::ThreadStoreId {
5937            "metadata-missing-store".to_string()
5938        }
5939
5940        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
5941            Ok(metadata)
5942        }
5943
5944        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
5945            Ok(Vec::new())
5946        }
5947
5948        async fn load_thread(
5949            &self,
5950            _thread_id: &ThreadId,
5951        ) -> anyhow::Result<Option<ThreadSnapshot>> {
5952            Ok(Some(ThreadSnapshot {
5953                metadata: None,
5954                ..ThreadSnapshot::default()
5955            }))
5956        }
5957
5958        async fn append_event(
5959            &self,
5960            _thread_id: &ThreadId,
5961            _envelope: &EventEnvelope,
5962        ) -> anyhow::Result<()> {
5963            Ok(())
5964        }
5965    }
5966
5967    struct MetadataMissingStoreFactory;
5968
5969    impl ThreadStoreFactory for MetadataMissingStoreFactory {
5970        fn id(&self) -> roder_api::thread::ThreadStoreId {
5971            "metadata-missing-store".to_string()
5972        }
5973
5974        fn create(&self) -> Arc<dyn ThreadStore> {
5975            Arc::new(MetadataMissingStore)
5976        }
5977    }
5978
5979    #[test]
5980    fn synthetic_app_server_events_are_not_thread_events() {
5981        for thread_id in ["app-server", "runtime", "thread-workflow"] {
5982            assert!(!should_persist_thread_event(thread_id));
5983        }
5984        assert!(should_persist_thread_event("thread-discovery"));
5985        assert!(should_persist_thread_event("thread-plan"));
5986        assert!(should_persist_thread_event("thread-process"));
5987        assert!(should_persist_thread_event("thread-1"));
5988    }
5989
5990    #[test]
5991    fn server_side_compaction_uses_catalog_ninety_percent_default() {
5992        assert_eq!(
5993            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
5994            Some(945_000)
5995        );
5996        assert_eq!(
5997            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
5998            Some(115_200)
5999        );
6000    }
6001
6002    #[test]
6003    fn server_side_compaction_respects_explicit_config_override() {
6004        let cfg = RuntimeConfig {
6005            auto_compact_token_limit: Some(123_456),
6006            ..RuntimeConfig::default()
6007        };
6008
6009        assert_eq!(
6010            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6011            Some(123_456)
6012        );
6013    }
6014
6015    #[tokio::test]
6016    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6017        let captured = Arc::new(StdMutex::new(None));
6018        let mut builder = ExtensionRegistryBuilder::new();
6019        builder.inference_engine(Arc::new(CapturingEngine {
6020            request: captured.clone(),
6021        }));
6022        let thread_root = std::env::temp_dir().join(format!(
6023            "roder-pre-request-compaction-{}",
6024            uuid::Uuid::new_v4()
6025        ));
6026        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6027            base_path: thread_root.clone(),
6028        }));
6029        let runtime = Arc::new(
6030            Runtime::new(
6031                builder.build().unwrap(),
6032                RuntimeConfig {
6033                    default_provider: PROVIDER_MOCK.to_string(),
6034                    default_model: "gpt-5.5".to_string(),
6035                    file_backed_dynamic_context: true,
6036                    ..RuntimeConfig::default()
6037                },
6038            )
6039            .unwrap(),
6040        );
6041        let thread_id = runtime
6042            .create_thread(Some("Pre-request compaction".to_string()))
6043            .await
6044            .unwrap()
6045            .thread_id;
6046        let old_turn = "old-turn".to_string();
6047        runtime
6048            .persist_turn_item(
6049                &thread_id,
6050                &old_turn,
6051                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6052            )
6053            .await
6054            .unwrap();
6055
6056        let mut events = runtime.subscribe_events();
6057        runtime
6058            .start_turn(StartTurnRequest {
6059                thread_id: thread_id.clone(),
6060                message: "continue".to_string(),
6061                images: Vec::new(),
6062                provider_override: None,
6063                model_override: None,
6064                reasoning_override: None,
6065                workspace: test_workspace(),
6066                instructions: InstructionBundle::default(),
6067                developer_context: None,
6068                task_ledger_required: false,
6069            })
6070            .await
6071            .unwrap();
6072        loop {
6073            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6074                .await
6075                .unwrap()
6076                .unwrap();
6077            if envelope.thread_id.as_deref() == Some(&thread_id)
6078                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6079            {
6080                break;
6081            }
6082        }
6083
6084        let request = captured.lock().unwrap().clone().unwrap();
6085        assert!(
6086            matches!(
6087                request.transcript.first(),
6088                Some(TranscriptItem::ContextCompaction(_))
6089            ),
6090            "provider request should start with a local emergency compaction item"
6091        );
6092        assert!(
6093            request.transcript.len() < 4,
6094            "provider request should not replay the full oversized prior transcript: {:?}",
6095            request.transcript
6096        );
6097
6098        let _ = std::fs::remove_dir_all(thread_root);
6099    }
6100
6101    #[tokio::test]
6102    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6103        let captured = Arc::new(StdMutex::new(None));
6104        let mut builder = ExtensionRegistryBuilder::new();
6105        builder.inference_engine(Arc::new(CapturingEngine {
6106            request: captured.clone(),
6107        }));
6108        let thread_root = std::env::temp_dir().join(format!(
6109            "roder-context-failure-continue-{}",
6110            uuid::Uuid::new_v4()
6111        ));
6112        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6113            base_path: thread_root.clone(),
6114        }));
6115        let runtime = Arc::new(
6116            Runtime::new(
6117                builder.build().unwrap(),
6118                RuntimeConfig {
6119                    default_provider: PROVIDER_MOCK.to_string(),
6120                    default_model: "gpt-5.5".to_string(),
6121                    file_backed_dynamic_context: true,
6122                    ..RuntimeConfig::default()
6123                },
6124            )
6125            .unwrap(),
6126        );
6127        let thread_id = runtime
6128            .create_thread(Some("Context failure continue".to_string()))
6129            .await
6130            .unwrap()
6131            .thread_id;
6132        let failed_turn = "failed-turn".to_string();
6133        runtime
6134            .persist_turn_item(
6135                &thread_id,
6136                &failed_turn,
6137                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6138            )
6139            .await
6140            .unwrap();
6141        runtime
6142            .persist_turn_item(
6143                &thread_id,
6144                &failed_turn,
6145                &TranscriptItem::Error(ErrorRecord {
6146                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6147                        .to_string(),
6148                }),
6149            )
6150            .await
6151            .unwrap();
6152
6153        let mut events = runtime.subscribe_events();
6154        runtime
6155            .start_turn(StartTurnRequest {
6156                thread_id: thread_id.clone(),
6157                message: "continue".to_string(),
6158                images: Vec::new(),
6159                provider_override: None,
6160                model_override: None,
6161                reasoning_override: None,
6162                workspace: test_workspace(),
6163                instructions: InstructionBundle::default(),
6164                developer_context: None,
6165                task_ledger_required: false,
6166            })
6167            .await
6168            .unwrap();
6169        loop {
6170            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6171                .await
6172                .unwrap()
6173                .unwrap();
6174            if envelope.thread_id.as_deref() == Some(&thread_id)
6175                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6176            {
6177                break;
6178            }
6179        }
6180
6181        let request = captured.lock().unwrap().clone().unwrap();
6182        assert!(
6183            matches!(
6184                request.transcript.first(),
6185                Some(TranscriptItem::ContextCompaction(_))
6186            ),
6187            "provider request after context-window failure should start with local compaction"
6188        );
6189        assert!(
6190            request
6191                .transcript
6192                .iter()
6193                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6194            "current continue prompt must be preserved: {:?}",
6195            request.transcript
6196        );
6197        assert!(
6198            !request.transcript.iter().any(
6199                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6200            ),
6201            "raw prior context-window error should be summarized, not replayed: {:?}",
6202            request.transcript
6203        );
6204
6205        let _ = std::fs::remove_dir_all(thread_root);
6206    }
6207
6208    #[tokio::test]
6209    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6210        let workspace = test_workspace();
6211        let mut builder = ExtensionRegistryBuilder::new();
6212        builder.inference_engine(Arc::new(FakeInferenceEngine));
6213        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6214        let runtime = Runtime::new(
6215            builder.build().unwrap(),
6216            RuntimeConfig {
6217                workspace: Some(workspace.clone()),
6218                ..RuntimeConfig::default()
6219            },
6220        )
6221        .unwrap();
6222
6223        let resolved = runtime
6224            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6225            .await
6226            .unwrap();
6227
6228        assert_eq!(resolved, workspace);
6229    }
6230
6231    #[tokio::test]
6232    async fn automations_can_create_project_thread_with_model_overrides() {
6233        let runtime = Runtime::fake().unwrap();
6234        let workspace = std::env::temp_dir().join("project");
6235        let metadata = runtime
6236            .create_thread_with(CreateThreadRequest {
6237                title: Some("Automation: nightly status".to_string()),
6238                workspace: workspace.display().to_string(),
6239                workspace_id: None,
6240                root_id: None,
6241                provider: Some("mock".to_string()),
6242                model: Some("mock".to_string()),
6243                selection_mode: None,
6244                tool_allowlist: Vec::new(),
6245                developer_instructions: None,
6246                external_tools: Vec::new(),
6247                runner: None,
6248            })
6249            .await
6250            .unwrap();
6251
6252        assert_eq!(
6253            metadata.title.as_deref(),
6254            Some("Automation: nightly status")
6255        );
6256        assert_eq!(metadata.workspace, workspace.display().to_string());
6257        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6258        assert_eq!(metadata.model.as_deref(), Some("mock"));
6259    }
6260
6261    #[tokio::test]
6262    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6263        let captured = Arc::new(StdMutex::new(None));
6264        let mut builder = ExtensionRegistryBuilder::new();
6265        builder.inference_engine(Arc::new(CapturingEngine {
6266            request: captured.clone(),
6267        }));
6268        let thread_root = std::env::temp_dir().join(format!(
6269            "roder-provider-compaction-boundary-{}",
6270            uuid::Uuid::new_v4()
6271        ));
6272        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6273            base_path: thread_root.clone(),
6274        }));
6275        let runtime = Arc::new(
6276            Runtime::new(
6277                builder.build().unwrap(),
6278                RuntimeConfig {
6279                    default_provider: PROVIDER_MOCK.to_string(),
6280                    default_model: "gpt-5.5".to_string(),
6281                    ..RuntimeConfig::default()
6282                },
6283            )
6284            .unwrap(),
6285        );
6286        let thread_id = runtime
6287            .create_thread(Some("Provider compaction boundary".to_string()))
6288            .await
6289            .unwrap()
6290            .thread_id;
6291        let old_turn = "old-turn".to_string();
6292        runtime
6293            .persist_turn_item(
6294                &thread_id,
6295                &old_turn,
6296                &TranscriptItem::UserMessage(UserMessage::text(
6297                    "old history that must not be replayed after provider compaction",
6298                )),
6299            )
6300            .await
6301            .unwrap();
6302        runtime
6303            .persist_turn_item(
6304                &thread_id,
6305                &old_turn,
6306                &TranscriptItem::AssistantMessage(AssistantMessage {
6307                    text: "old answer".to_string(),
6308                    phase: None,
6309                }),
6310            )
6311            .await
6312            .unwrap();
6313        runtime
6314            .persist_turn_item(
6315                &thread_id,
6316                &old_turn,
6317                &TranscriptItem::ProviderMetadata(serde_json::json!({
6318                    "output": [{
6319                        "id": "cmp_1",
6320                        "type": "compaction",
6321                        "encrypted_content": "opaque-state"
6322                    }]
6323                })),
6324            )
6325            .await
6326            .unwrap();
6327        runtime
6328            .persist_turn_item(
6329                &thread_id,
6330                &old_turn,
6331                &TranscriptItem::AssistantMessage(AssistantMessage {
6332                    text: "after compact".to_string(),
6333                    phase: None,
6334                }),
6335            )
6336            .await
6337            .unwrap();
6338
6339        let mut events = runtime.subscribe_events();
6340        runtime
6341            .start_turn(StartTurnRequest {
6342                thread_id: thread_id.clone(),
6343                message: "continue".to_string(),
6344                images: Vec::new(),
6345                provider_override: None,
6346                model_override: None,
6347                reasoning_override: None,
6348                workspace: test_workspace(),
6349                instructions: InstructionBundle::default(),
6350                developer_context: None,
6351                task_ledger_required: false,
6352            })
6353            .await
6354            .unwrap();
6355        loop {
6356            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6357                .await
6358                .unwrap()
6359                .unwrap();
6360            if envelope.thread_id.as_deref() == Some(&thread_id)
6361                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6362            {
6363                break;
6364            }
6365        }
6366
6367        let request = captured.lock().unwrap().clone().unwrap();
6368        let compaction_idx = request.transcript.iter().position(|item| {
6369            matches!(
6370                item,
6371                TranscriptItem::ProviderMetadata(metadata)
6372                    if crate::compaction::provider_metadata_has_compaction(metadata)
6373            )
6374        });
6375        assert!(
6376            compaction_idx.is_some(),
6377            "next provider request should include the provider compaction item: {:?}",
6378            request.transcript
6379        );
6380        // Skill/context injectors may prepend non-history items, but no prior-turn
6381        // conversation may appear before the latest provider compaction boundary.
6382        let history_before_boundary =
6383            request
6384                .transcript
6385                .iter()
6386                .take(compaction_idx.unwrap())
6387                .any(|item| match item {
6388                    TranscriptItem::AssistantMessage(_)
6389                    | TranscriptItem::ToolCall(_)
6390                    | TranscriptItem::ToolResult(_) => true,
6391                    TranscriptItem::UserMessage(message) => {
6392                        !message.text.contains("<skills>") && message.text != "continue"
6393                    }
6394                    _ => false,
6395                });
6396        assert!(
6397            !history_before_boundary,
6398            "pre-provider-compaction conversation must not be replayed: {:?}",
6399            request.transcript
6400        );
6401        assert!(
6402            !request.transcript.iter().any(|item| {
6403                matches!(
6404                    item,
6405                    TranscriptItem::UserMessage(message)
6406                        if message.text.contains("old history that must not be replayed")
6407                )
6408            }),
6409            "pre-provider-compaction history must not be replayed: {:?}",
6410            request.transcript
6411        );
6412        assert!(
6413            request.transcript.iter().any(|item| {
6414                matches!(
6415                    item,
6416                    TranscriptItem::UserMessage(message) if message.text == "continue"
6417                )
6418            }),
6419            "current continue prompt must be preserved: {:?}",
6420            request.transcript
6421        );
6422        assert_eq!(
6423            request.runtime.auto_compact_token_limit,
6424            Some(945_000),
6425            "gpt-5.5 should keep server-side auto compaction configured"
6426        );
6427
6428        let _ = std::fs::remove_dir_all(thread_root);
6429    }
6430
6431    #[test]
6432    fn server_side_compaction_is_only_enabled_for_supported_models() {
6433        let cfg = RuntimeConfig {
6434            auto_compact_token_limit: Some(123_456),
6435            ..RuntimeConfig::default()
6436        };
6437
6438        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6439        assert_eq!(
6440            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6441            None
6442        );
6443    }
6444
6445    #[test]
6446    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6447        let cfg = RuntimeConfig {
6448            reasoning: Some(REASONING_HIGH.to_string()),
6449            ..RuntimeConfig::default()
6450        };
6451
6452        assert_eq!(
6453            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6454            REASONING_NONE
6455        );
6456        assert_eq!(
6457            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6458            ReasoningConfig::default()
6459        );
6460    }
6461
6462    #[test]
6463    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6464        let cfg = RuntimeConfig {
6465            reasoning: Some(REASONING_MINIMAL.to_string()),
6466            ..RuntimeConfig::default()
6467        };
6468
6469        assert_eq!(
6470            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6471            REASONING_MEDIUM
6472        );
6473    }
6474
6475    #[test]
6476    fn unsupported_configured_gemini_reasoning_is_rejected() {
6477        let mut builder = ExtensionRegistryBuilder::new();
6478        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6479
6480        let err = match Runtime::new(
6481            builder.build().unwrap(),
6482            RuntimeConfig {
6483                default_model: "gemini-3.5-flash".to_string(),
6484                reasoning: Some(REASONING_XHIGH.to_string()),
6485                ..RuntimeConfig::default()
6486            },
6487        ) {
6488            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6489            Err(err) => err,
6490        };
6491
6492        assert!(
6493            err.to_string()
6494                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6495        );
6496    }
6497
6498    #[tokio::test]
6499    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6500        let runtime = Runtime::new(
6501            Runtime::fake().unwrap().registry,
6502            RuntimeConfig {
6503                reasoning: Some(REASONING_HIGH.to_string()),
6504                ..RuntimeConfig::default()
6505            },
6506        )
6507        .unwrap();
6508
6509        let cfg = runtime
6510            .select_provider(
6511                roder_api::catalog::PROVIDER_MOCK.to_string(),
6512                Some("claude-haiku-4-5-20251001".to_string()),
6513                Some(REASONING_NONE.to_string()),
6514            )
6515            .await
6516            .unwrap();
6517
6518        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6519        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6520    }
6521
6522    #[tokio::test]
6523    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6524        let runtime = Runtime::new(
6525            Runtime::fake().unwrap().registry,
6526            RuntimeConfig {
6527                reasoning: Some(REASONING_HIGH.to_string()),
6528                ..RuntimeConfig::default()
6529            },
6530        )
6531        .unwrap();
6532
6533        let cfg = runtime
6534            .select_provider(
6535                roder_api::catalog::PROVIDER_MOCK.to_string(),
6536                Some("mock".to_string()),
6537                Some(REASONING_NONE.to_string()),
6538            )
6539            .await
6540            .unwrap();
6541
6542        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6543    }
6544
6545    #[test]
6546    fn parallel_tool_calls_default_on_with_model_override() {
6547        assert!(parallel_tool_calls_for_model(
6548            &RuntimeConfig::default(),
6549            "custom-model"
6550        ));
6551
6552        let cfg = RuntimeConfig {
6553            model_parallel_tool_calls: std::collections::HashMap::from([(
6554                "custom-model".to_string(),
6555                false,
6556            )]),
6557            ..RuntimeConfig::default()
6558        };
6559
6560        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6561        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6562    }
6563
6564    #[test]
6565    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6566        let cfg = RuntimeConfig {
6567            model_profiles: std::collections::HashMap::from([(
6568                "gpt-5.5".to_string(),
6569                test_model_profile("gpt-5.5"),
6570            )]),
6571            ..RuntimeConfig::default()
6572        };
6573
6574        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6575
6576        let cfg = RuntimeConfig {
6577            model_parallel_tool_calls: std::collections::HashMap::from([(
6578                "gpt-5.5".to_string(),
6579                true,
6580            )]),
6581            ..cfg
6582        };
6583
6584        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6585    }
6586
6587    struct CapturingEngine {
6588        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6589    }
6590
6591    #[async_trait::async_trait]
6592    impl InferenceEngine for CapturingEngine {
6593        fn id(&self) -> String {
6594            roder_api::catalog::PROVIDER_MOCK.to_string()
6595        }
6596
6597        fn capabilities(&self) -> InferenceCapabilities {
6598            InferenceCapabilities::coding_agent_default()
6599        }
6600
6601        async fn list_models(
6602            &self,
6603            _ctx: InferenceProviderContext<'_>,
6604        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6605            Ok(roder_api::catalog::models_for_provider(
6606                roder_api::catalog::PROVIDER_MOCK,
6607                true,
6608            ))
6609        }
6610
6611        async fn stream_turn(
6612            &self,
6613            _ctx: InferenceTurnContext<'_>,
6614            request: AgentInferenceRequest,
6615        ) -> anyhow::Result<InferenceEventStream> {
6616            *self.request.lock().unwrap() = Some(request);
6617            Ok(Box::pin(stream::iter(vec![
6618                Ok(InferenceEvent::MessageDelta(MessageDelta {
6619                    text: "done".to_string(),
6620                    phase: None,
6621                })),
6622                Ok(InferenceEvent::Completed(CompletionMetadata {
6623                    stop_reason: Some("stop".to_string()),
6624                    provider_response_id: None,
6625                })),
6626            ])))
6627        }
6628    }
6629
6630    struct RoutingCaptureEngine {
6631        id: &'static str,
6632        models: Vec<ModelDescriptor>,
6633        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6634    }
6635
6636    #[async_trait::async_trait]
6637    impl InferenceEngine for RoutingCaptureEngine {
6638        fn id(&self) -> String {
6639            self.id.to_string()
6640        }
6641
6642        fn capabilities(&self) -> InferenceCapabilities {
6643            InferenceCapabilities::coding_agent_default()
6644        }
6645
6646        async fn list_models(
6647            &self,
6648            _ctx: InferenceProviderContext<'_>,
6649        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6650            Ok(self.models.clone())
6651        }
6652
6653        async fn stream_turn(
6654            &self,
6655            _ctx: InferenceTurnContext<'_>,
6656            request: AgentInferenceRequest,
6657        ) -> anyhow::Result<InferenceEventStream> {
6658            self.requests.lock().unwrap().push(request);
6659            Ok(Box::pin(stream::iter(vec![
6660                Ok(InferenceEvent::MessageDelta(MessageDelta {
6661                    text: "routed".to_string(),
6662                    phase: None,
6663                })),
6664                Ok(InferenceEvent::Completed(CompletionMetadata {
6665                    stop_reason: Some("stop".to_string()),
6666                    provider_response_id: None,
6667                })),
6668            ])))
6669        }
6670    }
6671
6672    struct StaticRouter {
6673        id: &'static str,
6674        decision: InferenceRoutingDecision,
6675        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6676    }
6677
6678    #[async_trait::async_trait]
6679    impl InferenceRouter for StaticRouter {
6680        fn id(&self) -> String {
6681            self.id.to_string()
6682        }
6683
6684        async fn route(
6685            &self,
6686            context: InferenceRoutingContext,
6687        ) -> anyhow::Result<InferenceRoutingDecision> {
6688            self.contexts.lock().unwrap().push(context);
6689            Ok(self.decision.clone())
6690        }
6691    }
6692
6693    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6694        ModelDescriptor {
6695            id: id.to_string(),
6696            name: id.to_string(),
6697            context_window: Some(128_000),
6698            default_reasoning: supported_reasoning
6699                .first()
6700                .map(|effort| (*effort).to_string()),
6701            supported_reasoning: supported_reasoning
6702                .iter()
6703                .map(|effort| ReasoningEffortDescriptor {
6704                    effort: (*effort).to_string(),
6705                    description: format!("{effort} reasoning"),
6706                })
6707                .collect(),
6708        }
6709    }
6710
6711    struct TaskLedgerCompletionGateEngine {
6712        calls: StdMutex<u32>,
6713        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6714    }
6715
6716    #[async_trait::async_trait]
6717    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6718        fn id(&self) -> String {
6719            roder_api::catalog::PROVIDER_MOCK.to_string()
6720        }
6721
6722        fn capabilities(&self) -> InferenceCapabilities {
6723            InferenceCapabilities::coding_agent_default()
6724        }
6725
6726        async fn list_models(
6727            &self,
6728            _ctx: InferenceProviderContext<'_>,
6729        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6730            Ok(roder_api::catalog::models_for_provider(
6731                roder_api::catalog::PROVIDER_MOCK,
6732                true,
6733            ))
6734        }
6735
6736        async fn stream_turn(
6737            &self,
6738            _ctx: InferenceTurnContext<'_>,
6739            request: AgentInferenceRequest,
6740        ) -> anyhow::Result<InferenceEventStream> {
6741            self.requests.lock().unwrap().push(request);
6742            let mut calls = self.calls.lock().unwrap();
6743            *calls += 1;
6744            let events = match *calls {
6745                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6746                    id: "ledger-open".to_string(),
6747                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6748                    arguments: serde_json::json!({
6749                        "tasks": [
6750                            {
6751                                "id": "inspect",
6752                                "content": "Inspect local assets",
6753                                "status": "completed",
6754                                "evidence": "listed workspace"
6755                            },
6756                            {
6757                                "id": "write",
6758                                "content": "Write /app/result.txt",
6759                                "status": "pending"
6760                            }
6761                        ],
6762                        "requireCompletionEvidence": true
6763                    })
6764                    .to_string(),
6765                }))],
6766                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6767                    id: "ledger-complete".to_string(),
6768                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6769                    arguments: serde_json::json!({
6770                        "tasks": [
6771                            {
6772                                "id": "inspect",
6773                                "content": "Inspect local assets",
6774                                "status": "completed",
6775                                "evidence": "listed workspace"
6776                            },
6777                            {
6778                                "id": "write",
6779                                "content": "Write /app/result.txt",
6780                                "status": "completed",
6781                                "evidence": "wrote answer"
6782                            }
6783                        ],
6784                        "requireCompletionEvidence": true
6785                    })
6786                    .to_string(),
6787                }))],
6788                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6789                    text: "final".to_string(),
6790                    phase: None,
6791                }))],
6792            };
6793            Ok(Box::pin(stream::iter(events.into_iter().chain(
6794                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6795                    stop_reason: Some("stop".to_string()),
6796                    provider_response_id: None,
6797                }))),
6798            ))))
6799        }
6800    }
6801
6802    struct VerificationGateEngine {
6803        calls: StdMutex<u32>,
6804    }
6805
6806    #[async_trait::async_trait]
6807    impl InferenceEngine for VerificationGateEngine {
6808        fn id(&self) -> String {
6809            roder_api::catalog::PROVIDER_MOCK.to_string()
6810        }
6811
6812        fn capabilities(&self) -> InferenceCapabilities {
6813            InferenceCapabilities::coding_agent_default()
6814        }
6815
6816        async fn list_models(
6817            &self,
6818            _ctx: InferenceProviderContext<'_>,
6819        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6820            Ok(roder_api::catalog::models_for_provider(
6821                roder_api::catalog::PROVIDER_MOCK,
6822                true,
6823            ))
6824        }
6825
6826        async fn stream_turn(
6827            &self,
6828            _ctx: InferenceTurnContext<'_>,
6829            request: AgentInferenceRequest,
6830        ) -> anyhow::Result<InferenceEventStream> {
6831            let mut calls = self.calls.lock().unwrap();
6832            *calls += 1;
6833            let events = match *calls {
6834                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6835                    id: "write-1".to_string(),
6836                    name: "write_file".to_string(),
6837                    arguments: serde_json::json!({
6838                        "path": "src/lib.rs",
6839                        "content": "pub fn answer() -> u8 { 42 }\n"
6840                    })
6841                    .to_string(),
6842                }))],
6843                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6844                    text: "done too early".to_string(),
6845                    phase: None,
6846                }))],
6847                3 if request.transcript.iter().any(|item| {
6848                    matches!(
6849                        item,
6850                        TranscriptItem::UserMessage(message)
6851                            if message.text.contains("Verification gate blocked final completion")
6852                    )
6853                }) =>
6854                {
6855                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6856                        id: "verify-1".to_string(),
6857                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6858                        arguments: serde_json::json!({
6859                            "originalTask": "write code",
6860                            "changedFiles": ["src/lib.rs"],
6861                            "toolEvidence": ["write_file wrote src/lib.rs"],
6862                            "testsRun": ["cargo test -p roder-core verification_gate"],
6863                            "openGaps": [],
6864                            "status": "completed"
6865                        })
6866                        .to_string(),
6867                    }))]
6868                }
6869                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6870                    text: "verified final".to_string(),
6871                    phase: None,
6872                }))],
6873            };
6874            Ok(Box::pin(stream::iter(events.into_iter().chain(
6875                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6876                    stop_reason: Some("stop".to_string()),
6877                    provider_response_id: None,
6878                }))),
6879            ))))
6880        }
6881    }
6882
6883    struct SpeedPolicyEngine {
6884        calls: StdMutex<u32>,
6885        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6886    }
6887
6888    #[async_trait::async_trait]
6889    impl InferenceEngine for SpeedPolicyEngine {
6890        fn id(&self) -> String {
6891            roder_api::catalog::PROVIDER_MOCK.to_string()
6892        }
6893
6894        fn capabilities(&self) -> InferenceCapabilities {
6895            InferenceCapabilities::coding_agent_default()
6896        }
6897
6898        async fn list_models(
6899            &self,
6900            _ctx: InferenceProviderContext<'_>,
6901        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6902            Ok(roder_api::catalog::models_for_provider(
6903                roder_api::catalog::PROVIDER_MOCK,
6904                true,
6905            ))
6906        }
6907
6908        async fn stream_turn(
6909            &self,
6910            _ctx: InferenceTurnContext<'_>,
6911            request: AgentInferenceRequest,
6912        ) -> anyhow::Result<InferenceEventStream> {
6913            self.requests.lock().unwrap().push(request.clone());
6914            let mut calls = self.calls.lock().unwrap();
6915            *calls += 1;
6916            let events = match *calls {
6917                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6918                    id: "write-1".to_string(),
6919                    name: "write_file".to_string(),
6920                    arguments: serde_json::json!({
6921                        "path": "src/lib.rs",
6922                        "content": "pub fn answer() -> u8 { 42 }\n"
6923                    })
6924                    .to_string(),
6925                }))],
6926                3 if request.transcript.iter().any(|item| {
6927                    matches!(
6928                        item,
6929                        TranscriptItem::UserMessage(message)
6930                            if message.text.contains("Verification gate blocked final completion")
6931                    )
6932                }) =>
6933                {
6934                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6935                        id: "verify-1".to_string(),
6936                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6937                        arguments: serde_json::json!({
6938                            "originalTask": "write code",
6939                            "changedFiles": ["src/lib.rs"],
6940                            "toolEvidence": ["write_file wrote src/lib.rs"],
6941                            "testsRun": ["cargo test -p roder-core speed_policy"],
6942                            "openGaps": [],
6943                            "status": "completed"
6944                        })
6945                        .to_string(),
6946                    }))]
6947                }
6948                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6949                    text: "done".to_string(),
6950                    phase: None,
6951                }))],
6952            };
6953            Ok(Box::pin(stream::iter(events.into_iter().chain(
6954                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6955                    stop_reason: Some("stop".to_string()),
6956                    provider_response_id: None,
6957                }))),
6958            ))))
6959        }
6960    }
6961
6962    struct SwitchCaptureEngine {
6963        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6964    }
6965
6966    #[async_trait::async_trait]
6967    impl InferenceEngine for SwitchCaptureEngine {
6968        fn id(&self) -> String {
6969            roder_api::catalog::PROVIDER_MOCK.to_string()
6970        }
6971
6972        fn capabilities(&self) -> InferenceCapabilities {
6973            InferenceCapabilities::coding_agent_default()
6974        }
6975
6976        async fn list_models(
6977            &self,
6978            _ctx: InferenceProviderContext<'_>,
6979        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6980            Ok(roder_api::catalog::models_for_provider(
6981                roder_api::catalog::PROVIDER_MOCK,
6982                true,
6983            ))
6984        }
6985
6986        async fn stream_turn(
6987            &self,
6988            _ctx: InferenceTurnContext<'_>,
6989            request: AgentInferenceRequest,
6990        ) -> anyhow::Result<InferenceEventStream> {
6991            self.requests.lock().unwrap().push(request);
6992            Ok(Box::pin(stream::iter(vec![
6993                Ok(InferenceEvent::MessageDelta(MessageDelta {
6994                    text: "done".to_string(),
6995                    phase: None,
6996                })),
6997                Ok(InferenceEvent::Completed(CompletionMetadata {
6998                    stop_reason: Some("stop".to_string()),
6999                    provider_response_id: None,
7000                })),
7001            ])))
7002        }
7003    }
7004
7005    struct DeadlineEngine;
7006
7007    #[async_trait::async_trait]
7008    impl InferenceEngine for DeadlineEngine {
7009        fn id(&self) -> String {
7010            roder_api::catalog::PROVIDER_MOCK.to_string()
7011        }
7012
7013        fn capabilities(&self) -> InferenceCapabilities {
7014            InferenceCapabilities::coding_agent_default()
7015        }
7016
7017        async fn list_models(
7018            &self,
7019            _ctx: InferenceProviderContext<'_>,
7020        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7021            Ok(Vec::new())
7022        }
7023
7024        async fn stream_turn(
7025            &self,
7026            _ctx: InferenceTurnContext<'_>,
7027            _request: AgentInferenceRequest,
7028        ) -> anyhow::Result<InferenceEventStream> {
7029            Ok(Box::pin(stream::once(async {
7030                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7031                Ok(InferenceEvent::MessageDelta(MessageDelta {
7032                    text: "too late".to_string(),
7033                    phase: None,
7034                }))
7035            })))
7036        }
7037    }
7038
7039    struct WriteFileContributor;
7040
7041    impl ToolContributor for WriteFileContributor {
7042        fn id(&self) -> String {
7043            "test-write".to_string()
7044        }
7045
7046        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7047            registry.register(Arc::new(WriteFileTool))
7048        }
7049    }
7050
7051    struct WriteFileTool;
7052
7053    #[async_trait::async_trait]
7054    impl ToolExecutor for WriteFileTool {
7055        fn spec(&self) -> ToolSpec {
7056            ToolSpec {
7057                name: "write_file".to_string(),
7058                description: "Write a test file.".to_string(),
7059                parameters: serde_json::json!({
7060                    "type": "object",
7061                    "properties": {
7062                        "path": { "type": "string" },
7063                        "content": { "type": "string" }
7064                    },
7065                    "required": ["path", "content"],
7066                    "additionalProperties": false
7067                }),
7068            }
7069        }
7070
7071        async fn execute(
7072            &self,
7073            _ctx: ToolExecutionContext,
7074            call: ToolCall,
7075        ) -> anyhow::Result<ToolResult> {
7076            let path = call
7077                .arguments
7078                .get("path")
7079                .and_then(serde_json::Value::as_str)
7080                .unwrap_or("src/lib.rs");
7081            Ok(ToolResult {
7082                id: call.id,
7083                name: call.name,
7084                text: format!("wrote {path}"),
7085                data: serde_json::json!({ "path": path }),
7086                is_error: false,
7087            })
7088        }
7089    }
7090
7091    struct ProfileToolContributor;
7092
7093    impl ToolContributor for ProfileToolContributor {
7094        fn id(&self) -> String {
7095            "profile-tools".to_string()
7096        }
7097
7098        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7099            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7100                registry.register(Arc::new(ProfileTool {
7101                    name: name.to_string(),
7102                }))?;
7103            }
7104            Ok(())
7105        }
7106    }
7107
7108    struct ProfileTool {
7109        name: String,
7110    }
7111
7112    #[async_trait::async_trait]
7113    impl ToolExecutor for ProfileTool {
7114        fn spec(&self) -> ToolSpec {
7115            ToolSpec {
7116                name: self.name.clone(),
7117                description: format!("{} test tool", self.name),
7118                parameters: serde_json::json!({
7119                    "type": "object",
7120                    "properties": {
7121                        "path": { "type": "string" },
7122                        "content": { "type": "string" }
7123                    },
7124                    "required": ["path", "content"],
7125                    "additionalProperties": false
7126                }),
7127            }
7128        }
7129
7130        async fn execute(
7131            &self,
7132            _ctx: ToolExecutionContext,
7133            call: ToolCall,
7134        ) -> anyhow::Result<ToolResult> {
7135            Ok(ToolResult {
7136                id: call.id,
7137                name: call.name,
7138                text: "ok".to_string(),
7139                data: serde_json::json!({}),
7140                is_error: false,
7141            })
7142        }
7143    }
7144
7145    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7146        ModelHarnessProfile {
7147            model: model.to_string(),
7148            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7149            provider_family: ProviderFamily::OpenAi,
7150            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7151            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7152            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7153            reasoning: ModelProfileReasoning {
7154                orientation: Some(REASONING_LOW.to_string()),
7155                execution: Some(REASONING_LOW.to_string()),
7156                verification: Some(REASONING_LOW.to_string()),
7157                recovery: Some(REASONING_LOW.to_string()),
7158            },
7159            parallel_tool_calls: Some(false),
7160            auto_compact_token_limit: Some(123_000),
7161        }
7162    }
7163
7164    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7165        let captured = Arc::new(StdMutex::new(None));
7166        let mut builder = ExtensionRegistryBuilder::new();
7167        builder.inference_engine(Arc::new(CapturingEngine {
7168            request: captured.clone(),
7169        }));
7170        builder.tool_contributor(Arc::new(ProfileToolContributor));
7171        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7172        let mut rx = runtime.subscribe_events();
7173        let turn_id = runtime
7174            .start_turn(StartTurnRequest {
7175                thread_id: "thread-model-profile".to_string(),
7176                message: "use profile knobs".to_string(),
7177                images: Vec::new(),
7178                provider_override: None,
7179                model_override: None,
7180                reasoning_override: None,
7181                workspace: test_workspace(),
7182                instructions: InstructionBundle {
7183                    system: None,
7184                    developer: Some("base developer".to_string()),
7185                    developer_context: None,
7186                },
7187                developer_context: None,
7188                task_ledger_required: false,
7189            })
7190            .await
7191            .unwrap();
7192
7193        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7194            loop {
7195                let envelope = rx.recv().await.unwrap();
7196                if envelope.turn_id.as_deref() != Some(&turn_id) {
7197                    continue;
7198                }
7199                match envelope.event {
7200                    RoderEvent::TurnCompleted(_) => break,
7201                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7202                    _ => {}
7203                }
7204            }
7205        })
7206        .await
7207        .unwrap();
7208
7209        captured.lock().unwrap().clone().unwrap()
7210    }
7211
7212    struct ToolThenStopEngine {
7213        calls: StdMutex<u32>,
7214    }
7215
7216    #[async_trait::async_trait]
7217    impl InferenceEngine for ToolThenStopEngine {
7218        fn id(&self) -> String {
7219            roder_api::catalog::PROVIDER_MOCK.to_string()
7220        }
7221
7222        fn capabilities(&self) -> InferenceCapabilities {
7223            InferenceCapabilities::coding_agent_default()
7224        }
7225
7226        async fn list_models(
7227            &self,
7228            _ctx: InferenceProviderContext<'_>,
7229        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7230            Ok(roder_api::catalog::models_for_provider(
7231                roder_api::catalog::PROVIDER_MOCK,
7232                true,
7233            ))
7234        }
7235
7236        async fn stream_turn(
7237            &self,
7238            _ctx: InferenceTurnContext<'_>,
7239            _request: AgentInferenceRequest,
7240        ) -> anyhow::Result<InferenceEventStream> {
7241            let mut calls = self.calls.lock().unwrap();
7242            *calls += 1;
7243            let events = match *calls {
7244                1 => vec![
7245                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7246                        id: "write-1".to_string(),
7247                        name: "write_file".to_string(),
7248                        arguments: serde_json::json!({
7249                            "path": "src/lib.rs",
7250                            "content": "pub fn answer() -> u8 { 42 }\n"
7251                        })
7252                        .to_string(),
7253                    })),
7254                    Ok(InferenceEvent::Completed(CompletionMetadata {
7255                        stop_reason: Some("tool_use".to_string()),
7256                        provider_response_id: None,
7257                    })),
7258                ],
7259                _ => vec![
7260                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7261                        text: "final".to_string(),
7262                        phase: None,
7263                    })),
7264                    Ok(InferenceEvent::Completed(CompletionMetadata {
7265                        stop_reason: Some("end_turn".to_string()),
7266                        provider_response_id: None,
7267                    })),
7268                ],
7269            };
7270            Ok(Box::pin(stream::iter(events)))
7271        }
7272    }
7273
7274    #[tokio::test]
7275    async fn turn_completed_reports_terminal_step_finish_reason() {
7276        let mut builder = ExtensionRegistryBuilder::new();
7277        builder.inference_engine(Arc::new(ToolThenStopEngine {
7278            calls: StdMutex::new(0),
7279        }));
7280        builder.tool_contributor(Arc::new(WriteFileContributor));
7281        let runtime = Arc::new(
7282            Runtime::new(
7283                builder.build().unwrap(),
7284                RuntimeConfig {
7285                    policy_mode: PolicyMode::Bypass,
7286                    agent_swarm_mode: false,
7287                    ..RuntimeConfig::default()
7288                },
7289            )
7290            .unwrap(),
7291        );
7292        let mut rx = runtime.subscribe_events();
7293        let turn_id = runtime
7294            .start_turn(StartTurnRequest {
7295                thread_id: "thread-finish-reason".to_string(),
7296                message: "write then finish".to_string(),
7297                images: Vec::new(),
7298                provider_override: None,
7299                model_override: None,
7300                reasoning_override: None,
7301                workspace: test_workspace(),
7302                instructions: InstructionBundle {
7303                    system: None,
7304                    developer: None,
7305                    developer_context: None,
7306                },
7307                developer_context: None,
7308                task_ledger_required: false,
7309            })
7310            .await
7311            .unwrap();
7312
7313        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7314            loop {
7315                let envelope = rx.recv().await.unwrap();
7316                if envelope.turn_id.as_deref() != Some(&turn_id) {
7317                    continue;
7318                }
7319                match envelope.event {
7320                    RoderEvent::TurnCompleted(event) => break event,
7321                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7322                    _ => {}
7323                }
7324            }
7325        })
7326        .await
7327        .unwrap();
7328
7329        // The mid-turn tool_use stop reason must not leak; the terminal
7330        // end_turn step decides the turn's finish reason.
7331        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7332    }
7333
7334    #[tokio::test]
7335    async fn inference_router_selection_changes_request_model_and_records_event() {
7336        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7337        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7338        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7339        let selected = ModelSelection {
7340            provider: "routed-provider".to_string(),
7341            model: "routed-model".to_string(),
7342        };
7343        let default = ModelSelection {
7344            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7345            model: "mock".to_string(),
7346        };
7347        let decision = InferenceRoutingDecision {
7348            reasoning: Some(ReasoningConfig {
7349                enabled: true,
7350                level: Some(REASONING_LOW.to_string()),
7351            }),
7352            confidence: Some(0.91),
7353            baseline: Some(default.clone()),
7354            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7355                "intent", "routine",
7356            )],
7357            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7358        };
7359
7360        let mut builder = ExtensionRegistryBuilder::new();
7361        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7362            id: roder_api::catalog::PROVIDER_MOCK,
7363            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7364            requests: default_requests.clone(),
7365        }));
7366        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7367            id: "routed-provider",
7368            models: vec![routing_test_model(
7369                "routed-model",
7370                &[REASONING_LOW, REASONING_MEDIUM],
7371            )],
7372            requests: routed_requests.clone(),
7373        }));
7374        builder.inference_router(Arc::new(StaticRouter {
7375            id: "test-router",
7376            decision,
7377            contexts: contexts.clone(),
7378        }));
7379        let thread_root =
7380            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7381        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7382            base_path: thread_root.clone(),
7383        }));
7384        let runtime = Arc::new(
7385            Runtime::new(
7386                builder.build().unwrap(),
7387                RuntimeConfig {
7388                    default_provider: default.provider.clone(),
7389                    default_model: default.model.clone(),
7390                    ..RuntimeConfig::default()
7391                },
7392            )
7393            .unwrap(),
7394        );
7395        let thread_id = runtime
7396            .create_thread_with(CreateThreadRequest {
7397                title: Some("Routing auto".to_string()),
7398                workspace: test_workspace(),
7399                workspace_id: None,
7400                root_id: None,
7401                provider: Some(default.provider.clone()),
7402                model: Some(default.model.clone()),
7403                tool_allowlist: Vec::new(),
7404                developer_instructions: None,
7405                external_tools: Vec::new(),
7406                selection_mode: Some(ModelSelectionMode::auto(
7407                    "test-router:coding",
7408                    "test-router",
7409                    "Auto: Coding",
7410                    default.clone(),
7411                    Some("coding".to_string()),
7412                    None,
7413                )),
7414                runner: None,
7415            })
7416            .await
7417            .unwrap()
7418            .thread_id;
7419        let mut rx = runtime.subscribe_events();
7420        let turn_id = runtime
7421            .start_turn(StartTurnRequest {
7422                thread_id: thread_id.clone(),
7423                message: "small cleanup".to_string(),
7424                images: Vec::new(),
7425                provider_override: None,
7426                model_override: None,
7427                reasoning_override: None,
7428                workspace: test_workspace(),
7429                instructions: InstructionBundle::default(),
7430                developer_context: None,
7431                task_ledger_required: false,
7432            })
7433            .await
7434            .unwrap();
7435
7436        let mut routing_event = None;
7437        let mut inference_started = None;
7438        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7439            loop {
7440                let envelope = rx.recv().await.unwrap();
7441                if envelope.turn_id.as_deref() != Some(&turn_id) {
7442                    continue;
7443                }
7444                match envelope.event {
7445                    RoderEvent::InferenceRoutingDecision(event) => {
7446                        routing_event = Some(event);
7447                    }
7448                    RoderEvent::InferenceStarted(event) => {
7449                        inference_started = Some(event);
7450                    }
7451                    RoderEvent::TurnCompleted(_) => break,
7452                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7453                    _ => {}
7454                }
7455            }
7456        })
7457        .await
7458        .unwrap();
7459
7460        assert!(default_requests.lock().unwrap().is_empty());
7461        let routed_requests = routed_requests.lock().unwrap();
7462        assert_eq!(routed_requests.len(), 1);
7463        assert_eq!(routed_requests[0].model, selected);
7464        assert_eq!(
7465            routed_requests[0].reasoning.level.as_deref(),
7466            Some(REASONING_LOW)
7467        );
7468        assert_eq!(
7469            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7470            "selected"
7471        );
7472
7473        let routing_event = routing_event.expect("routing decision event");
7474        assert_eq!(routing_event.default_selection, default);
7475        assert_eq!(routing_event.selected_selection, selected);
7476        assert_eq!(
7477            routing_event.decision.outcome,
7478            InferenceRoutingOutcome::Selected
7479        );
7480        assert_eq!(
7481            inference_started.expect("inference started event").model,
7482            selected
7483        );
7484
7485        let contexts = contexts.lock().unwrap();
7486        assert_eq!(contexts.len(), 1);
7487        assert_eq!(contexts[0].default_selection, default);
7488        assert_eq!(contexts[0].candidates.len(), 2);
7489        assert!(
7490            contexts[0]
7491                .signals
7492                .iter()
7493                .any(|signal| signal.key == "profile" && signal.value == "coding")
7494        );
7495        let _ = std::fs::remove_dir_all(thread_root);
7496    }
7497
7498    #[tokio::test]
7499    async fn inference_router_is_bypassed_for_explicit_selection() {
7500        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7501        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7502        let selected = ModelSelection {
7503            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7504            model: "mock".to_string(),
7505        };
7506
7507        let mut builder = ExtensionRegistryBuilder::new();
7508        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7509            id: roder_api::catalog::PROVIDER_MOCK,
7510            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7511            requests: requests.clone(),
7512        }));
7513        builder.inference_router(Arc::new(StaticRouter {
7514            id: "test-router",
7515            decision: InferenceRoutingDecision::selected(
7516                "test-router",
7517                ModelSelection {
7518                    provider: "missing".to_string(),
7519                    model: "missing".to_string(),
7520                },
7521                "would route if called",
7522            ),
7523            contexts: contexts.clone(),
7524        }));
7525        let thread_root =
7526            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7527        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7528            base_path: thread_root.clone(),
7529        }));
7530        let runtime = Arc::new(
7531            Runtime::new(
7532                builder.build().unwrap(),
7533                RuntimeConfig {
7534                    default_provider: selected.provider.clone(),
7535                    default_model: selected.model.clone(),
7536                    inference_router: RuntimeInferenceRouterConfig {
7537                        enabled: true,
7538                        router_id: Some("test-router".to_string()),
7539                    },
7540                    ..RuntimeConfig::default()
7541                },
7542            )
7543            .unwrap(),
7544        );
7545        let thread_id = runtime
7546            .create_thread_with(CreateThreadRequest {
7547                title: Some("Routing explicit".to_string()),
7548                workspace: test_workspace(),
7549                workspace_id: None,
7550                root_id: None,
7551                provider: Some(selected.provider.clone()),
7552                model: Some(selected.model.clone()),
7553                tool_allowlist: Vec::new(),
7554                developer_instructions: None,
7555                external_tools: Vec::new(),
7556                selection_mode: Some(ModelSelectionMode::auto(
7557                    "test-router:default",
7558                    "test-router",
7559                    "Auto",
7560                    selected.clone(),
7561                    None,
7562                    None,
7563                )),
7564                runner: None,
7565            })
7566            .await
7567            .unwrap()
7568            .thread_id;
7569        let mut rx = runtime.subscribe_events();
7570        let turn_id = runtime
7571            .start_turn(StartTurnRequest {
7572                thread_id,
7573                message: "use explicit selection".to_string(),
7574                images: Vec::new(),
7575                provider_override: Some(selected.provider.clone()),
7576                model_override: Some(selected.model.clone()),
7577                reasoning_override: None,
7578                workspace: test_workspace(),
7579                instructions: InstructionBundle::default(),
7580                developer_context: None,
7581                task_ledger_required: false,
7582            })
7583            .await
7584            .unwrap();
7585
7586        let mut saw_routing_event = false;
7587        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7588            loop {
7589                let envelope = rx.recv().await.unwrap();
7590                if envelope.turn_id.as_deref() != Some(&turn_id) {
7591                    continue;
7592                }
7593                match envelope.event {
7594                    RoderEvent::InferenceRoutingDecision(_) => {
7595                        saw_routing_event = true;
7596                    }
7597                    RoderEvent::TurnCompleted(_) => break,
7598                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7599                    _ => {}
7600                }
7601            }
7602        })
7603        .await
7604        .unwrap();
7605
7606        assert!(!saw_routing_event);
7607        assert!(contexts.lock().unwrap().is_empty());
7608        let requests = requests.lock().unwrap();
7609        assert_eq!(requests.len(), 1);
7610        assert_eq!(requests[0].model, selected);
7611        let _ = std::fs::remove_dir_all(thread_root);
7612    }
7613
7614    #[tokio::test]
7615    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7616        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7617        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7618        let selected = ModelSelection {
7619            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7620            model: "mock".to_string(),
7621        };
7622
7623        let mut builder = ExtensionRegistryBuilder::new();
7624        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7625            id: roder_api::catalog::PROVIDER_MOCK,
7626            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7627            requests: requests.clone(),
7628        }));
7629        builder.inference_router(Arc::new(StaticRouter {
7630            id: "test-router",
7631            decision: InferenceRoutingDecision::selected(
7632                "test-router",
7633                ModelSelection {
7634                    provider: "missing".to_string(),
7635                    model: "missing".to_string(),
7636                },
7637                "would route if called",
7638            ),
7639            contexts: contexts.clone(),
7640        }));
7641        let thread_root =
7642            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7643        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7644            base_path: thread_root.clone(),
7645        }));
7646        let runtime = Arc::new(
7647            Runtime::new(
7648                builder.build().unwrap(),
7649                RuntimeConfig {
7650                    default_provider: selected.provider.clone(),
7651                    default_model: selected.model.clone(),
7652                    inference_router: RuntimeInferenceRouterConfig {
7653                        enabled: true,
7654                        router_id: Some("test-router".to_string()),
7655                    },
7656                    ..RuntimeConfig::default()
7657                },
7658            )
7659            .unwrap(),
7660        );
7661        let thread_id = runtime
7662            .create_thread_with(CreateThreadRequest {
7663                title: Some("Routing manual".to_string()),
7664                workspace: test_workspace(),
7665                workspace_id: None,
7666                root_id: None,
7667                provider: Some(selected.provider.clone()),
7668                model: Some(selected.model.clone()),
7669                tool_allowlist: Vec::new(),
7670                developer_instructions: None,
7671                external_tools: Vec::new(),
7672                selection_mode: Some(ModelSelectionMode::manual(
7673                    selected.provider.clone(),
7674                    selected.model.clone(),
7675                    None,
7676                )),
7677                runner: None,
7678            })
7679            .await
7680            .unwrap()
7681            .thread_id;
7682        let mut rx = runtime.subscribe_events();
7683        let turn_id = runtime
7684            .start_turn(StartTurnRequest {
7685                thread_id,
7686                message: "use selected manual model".to_string(),
7687                images: Vec::new(),
7688                provider_override: None,
7689                model_override: None,
7690                reasoning_override: None,
7691                workspace: test_workspace(),
7692                instructions: InstructionBundle::default(),
7693                developer_context: None,
7694                task_ledger_required: false,
7695            })
7696            .await
7697            .unwrap();
7698
7699        let mut saw_routing_event = false;
7700        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7701            loop {
7702                let envelope = rx.recv().await.unwrap();
7703                if envelope.turn_id.as_deref() != Some(&turn_id) {
7704                    continue;
7705                }
7706                match envelope.event {
7707                    RoderEvent::InferenceRoutingDecision(_) => {
7708                        saw_routing_event = true;
7709                    }
7710                    RoderEvent::TurnCompleted(_) => break,
7711                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7712                    _ => {}
7713                }
7714            }
7715        })
7716        .await
7717        .unwrap();
7718
7719        assert!(!saw_routing_event);
7720        assert!(contexts.lock().unwrap().is_empty());
7721        let requests = requests.lock().unwrap();
7722        assert_eq!(requests.len(), 1);
7723        assert_eq!(requests[0].model, selected);
7724        let _ = std::fs::remove_dir_all(thread_root);
7725    }
7726
7727    #[test]
7728    fn enabled_inference_router_requires_registered_router() {
7729        let mut builder = ExtensionRegistryBuilder::new();
7730        builder.inference_engine(Arc::new(FakeInferenceEngine));
7731
7732        let err = match Runtime::new(
7733            builder.build().unwrap(),
7734            RuntimeConfig {
7735                inference_router: RuntimeInferenceRouterConfig {
7736                    enabled: true,
7737                    router_id: Some("missing-router".to_string()),
7738                },
7739                ..RuntimeConfig::default()
7740            },
7741        ) {
7742            Ok(_) => panic!("runtime should reject unknown inference router"),
7743            Err(err) => err,
7744        };
7745
7746        assert!(
7747            err.to_string()
7748                .contains("inference router \"missing-router\" is not registered")
7749        );
7750    }
7751
7752    #[tokio::test]
7753    async fn model_profile_routes_request_knobs_to_next_inference() {
7754        let request = captured_profile_request(RuntimeConfig {
7755            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7756            default_model: "gpt-5.5".to_string(),
7757            model_profiles: std::collections::HashMap::from([(
7758                "gpt-5.5".to_string(),
7759                test_model_profile("gpt-5.5"),
7760            )]),
7761            ..RuntimeConfig::default()
7762        })
7763        .await;
7764
7765        let tool_names = request
7766            .tools
7767            .iter()
7768            .map(|tool| tool.name.as_str())
7769            .collect::<Vec<_>>();
7770        assert!(tool_names.contains(&"apply_patch"));
7771        assert!(tool_names.contains(&"edit"));
7772        assert!(tool_names.contains(&"multi_edit"));
7773        assert!(tool_names.contains(&"write_file"));
7774        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7775        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7776        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7777        assert!(
7778            request
7779                .instructions
7780                .developer
7781                .as_deref()
7782                .unwrap_or_default()
7783                .contains("Use the provided context as the current working set")
7784        );
7785        assert_eq!(
7786            request
7787                .metadata
7788                .pointer("/modelProfile/schemaPolicy")
7789                .and_then(serde_json::Value::as_str),
7790            Some("standard_required_first")
7791        );
7792    }
7793
7794    #[tokio::test]
7795    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7796        let request = captured_profile_request(RuntimeConfig {
7797            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7798            default_model: "gpt-5.6-sol".to_string(),
7799            reasoning: Some(REASONING_ULTRA.to_string()),
7800            ..RuntimeConfig::default()
7801        })
7802        .await;
7803
7804        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
7805        let developer = request
7806            .instructions
7807            .developer
7808            .as_deref()
7809            .expect("ultra developer instructions");
7810        assert!(developer.contains("Proactive multi-agent delegation is active"));
7811    }
7812
7813    #[tokio::test]
7814    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
7815        let request = captured_profile_request(RuntimeConfig {
7816            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7817            default_model: "gpt-5.6-sol".to_string(),
7818            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
7819            ..RuntimeConfig::default()
7820        })
7821        .await;
7822
7823        let developer = request
7824            .instructions
7825            .developer
7826            .as_deref()
7827            .expect("sol developer instructions");
7828        assert!(developer.contains("Do not spawn sub-agents unless"));
7829        assert!(!developer.contains("Proactive multi-agent delegation is active"));
7830    }
7831
7832    #[tokio::test]
7833    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
7834        let request = captured_profile_request(RuntimeConfig {
7835            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7836            default_model: "gpt-5.6-luna".to_string(),
7837            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
7838            ..RuntimeConfig::default()
7839        })
7840        .await;
7841
7842        let developer = request
7843            .instructions
7844            .developer
7845            .as_deref()
7846            .unwrap_or_default();
7847        assert!(!developer.contains("Do not spawn sub-agents unless"));
7848        assert!(!developer.contains("Proactive multi-agent delegation is active"));
7849    }
7850
7851    #[tokio::test]
7852    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
7853        let captured = Arc::new(StdMutex::new(None));
7854        let mut builder = ExtensionRegistryBuilder::new();
7855        builder.inference_engine(Arc::new(CapturingEngine {
7856            request: captured.clone(),
7857        }));
7858        let runtime = Arc::new(
7859            Runtime::new(
7860                builder.build().unwrap(),
7861                RuntimeConfig {
7862                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7863                    default_model: "gpt-5.5".to_string(),
7864                    ..RuntimeConfig::default()
7865                },
7866            )
7867            .unwrap(),
7868        );
7869
7870        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
7871            let mut rx = runtime.subscribe_events();
7872            let turn_id = runtime
7873                .start_turn(StartTurnRequest {
7874                    thread_id: "thread-turn-context".to_string(),
7875                    message: "hello".to_string(),
7876                    images: Vec::new(),
7877                    provider_override: None,
7878                    model_override: None,
7879                    reasoning_override: None,
7880                    workspace: test_workspace(),
7881                    instructions: InstructionBundle::default(),
7882                    developer_context,
7883                    task_ledger_required: false,
7884                })
7885                .await
7886                .unwrap();
7887            tokio::time::timeout(std::time::Duration::from_secs(5), async {
7888                loop {
7889                    let envelope = rx.recv().await.unwrap();
7890                    if envelope.turn_id.as_deref() != Some(&turn_id) {
7891                        continue;
7892                    }
7893                    match envelope.event {
7894                        RoderEvent::TurnCompleted(_) => break,
7895                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7896                        _ => {}
7897                    }
7898                }
7899            })
7900            .await
7901            .unwrap();
7902        }
7903
7904        run_turn(
7905            &runtime,
7906            Some("Connected accounts: example-service.".to_string()),
7907        )
7908        .await;
7909        let request = captured.lock().unwrap().clone().unwrap();
7910        assert_eq!(
7911            request.instructions.developer_context.as_deref(),
7912            Some("Connected accounts: example-service.")
7913        );
7914
7915        // The context is per-turn only: the next turn on the same thread
7916        // without a developerContext must not see the previous one.
7917        run_turn(&runtime, None).await;
7918        let request = captured.lock().unwrap().clone().unwrap();
7919        assert_eq!(request.instructions.developer_context, None);
7920    }
7921
7922    #[tokio::test]
7923    async fn tool_search_overrides_route_to_next_inference_request() {
7924        let request = captured_profile_request(RuntimeConfig {
7925            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7926            default_model: "gpt-5.4".to_string(),
7927            tool_search: ToolSearchConfig {
7928                mode: roder_api::inference::ToolSearchMode::Auto,
7929                max_catalog_items: Some(100),
7930                ..ToolSearchConfig::default()
7931            },
7932            provider_tool_search: std::collections::HashMap::from([(
7933                roder_api::catalog::PROVIDER_MOCK.to_string(),
7934                roder_api::inference::ToolSearchConfigOverlay {
7935                    include_skills: Some(false),
7936                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
7937                    ..Default::default()
7938                },
7939            )]),
7940            model_tool_search: std::collections::HashMap::from([(
7941                "gpt-5.4".to_string(),
7942                roder_api::inference::ToolSearchConfigOverlay {
7943                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
7944                    max_catalog_items: Some(25),
7945                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
7946                    ..Default::default()
7947                },
7948            )]),
7949            ..RuntimeConfig::default()
7950        })
7951        .await;
7952
7953        assert_eq!(
7954            request.runtime.tool_search.mode,
7955            roder_api::inference::ToolSearchMode::ProviderNative
7956        );
7957        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
7958        assert_eq!(
7959            request.runtime.tool_search.provider_variant,
7960            roder_api::inference::ToolSearchProviderVariant::Bm25
7961        );
7962    }
7963
7964    #[tokio::test]
7965    async fn context_entrypoint_hints_use_turn_workspace() {
7966        let process_workspace = runtime_test_workspace("entrypoint-process");
7967        let thread_workspace = runtime_test_workspace("entrypoint-thread");
7968        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
7969        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
7970        std::fs::write(
7971            process_workspace.join("src/sidebar-thread-groups.ts"),
7972            "export const desktopLeak = true;\n",
7973        )
7974        .unwrap();
7975        std::fs::write(
7976            thread_workspace.join("src/voice-plan-feedback.ts"),
7977            "export const voicePlanFeedback = true;\n",
7978        )
7979        .unwrap();
7980
7981        let captured = Arc::new(StdMutex::new(None));
7982        let mut builder = ExtensionRegistryBuilder::new();
7983        builder.inference_engine(Arc::new(CapturingEngine {
7984            request: captured.clone(),
7985        }));
7986        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
7987            process_workspace.clone(),
7988        )));
7989        let runtime = Arc::new(
7990            Runtime::new(
7991                builder.build().unwrap(),
7992                RuntimeConfig {
7993                    workspace: Some(process_workspace.display().to_string()),
7994                    ..RuntimeConfig::default()
7995                },
7996            )
7997            .unwrap(),
7998        );
7999        let mut rx = runtime.subscribe_events();
8000
8001        let turn_id = runtime
8002            .start_turn(StartTurnRequest {
8003                thread_id: "thread-workspace-entrypoint".to_string(),
8004                message: "investigate voice plan feedback".to_string(),
8005                images: Vec::new(),
8006                provider_override: None,
8007                model_override: None,
8008                reasoning_override: None,
8009                workspace: thread_workspace.display().to_string(),
8010                instructions: crate::instructions::default_instructions(),
8011                developer_context: None,
8012                task_ledger_required: false,
8013            })
8014            .await
8015            .unwrap();
8016
8017        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8018            loop {
8019                let envelope = rx.recv().await.unwrap();
8020                if envelope.turn_id.as_deref() != Some(&turn_id) {
8021                    continue;
8022                }
8023                match envelope.event {
8024                    RoderEvent::TurnCompleted(_) => break,
8025                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8026                    _ => {}
8027                }
8028            }
8029        })
8030        .await
8031        .unwrap();
8032
8033        let request = captured.lock().unwrap().clone().expect("captured request");
8034        let transcript_text = request
8035            .transcript
8036            .iter()
8037            .map(|item| match item {
8038                TranscriptItem::UserMessage(message) => message.text.as_str(),
8039                _ => "",
8040            })
8041            .collect::<Vec<_>>()
8042            .join("\n");
8043        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8044        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8045
8046        let _ = std::fs::remove_dir_all(process_workspace);
8047        let _ = std::fs::remove_dir_all(thread_workspace);
8048    }
8049
8050    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8051        let path =
8052            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8053        let _ = std::fs::remove_dir_all(&path);
8054        std::fs::create_dir_all(&path).unwrap();
8055        path
8056    }
8057
8058    #[tokio::test]
8059    async fn model_profile_user_model_knobs_override_profile_defaults() {
8060        let request = captured_profile_request(RuntimeConfig {
8061            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8062            default_model: "gpt-5.5".to_string(),
8063            reasoning: Some(REASONING_HIGH.to_string()),
8064            auto_compact_token_limit: Some(999),
8065            model_edit_tools: std::collections::HashMap::from([(
8066                "gpt-5.5".to_string(),
8067                EDIT_TOOL_PATCH.to_string(),
8068            )]),
8069            model_parallel_tool_calls: std::collections::HashMap::from([(
8070                "gpt-5.5".to_string(),
8071                true,
8072            )]),
8073            model_profiles: std::collections::HashMap::from([(
8074                "gpt-5.5".to_string(),
8075                test_model_profile("gpt-5.5"),
8076            )]),
8077            ..RuntimeConfig::default()
8078        })
8079        .await;
8080
8081        let tool_names = request
8082            .tools
8083            .iter()
8084            .map(|tool| tool.name.as_str())
8085            .collect::<Vec<_>>();
8086        assert!(tool_names.contains(&"apply_patch"));
8087        assert!(!tool_names.contains(&"edit"));
8088        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8089        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8090        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8091    }
8092
8093    #[tokio::test]
8094    async fn runtime_tool_allowlist_filters_advertised_tools() {
8095        let request = captured_profile_request(RuntimeConfig {
8096            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8097            default_model: "gpt-5.5".to_string(),
8098            tool_allowlist: vec!["edit".to_string()],
8099            model_profiles: std::collections::HashMap::from([(
8100                "gpt-5.5".to_string(),
8101                test_model_profile("gpt-5.5"),
8102            )]),
8103            ..RuntimeConfig::default()
8104        })
8105        .await;
8106
8107        let tool_names = request
8108            .tools
8109            .iter()
8110            .map(|tool| tool.name.as_str())
8111            .collect::<Vec<_>>();
8112        assert_eq!(tool_names, vec!["edit"]);
8113    }
8114
8115    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8116    /// overrides and returns the captured inference request.
8117    async fn captured_thread_override_request(
8118        runtime: &Arc<Runtime>,
8119        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8120        tool_allowlist: Vec<String>,
8121        developer_instructions: Option<String>,
8122        external_tools: Vec<ToolSpec>,
8123    ) -> AgentInferenceRequest {
8124        let thread_id = runtime
8125            .create_thread_with(CreateThreadRequest {
8126                title: Some("Thread overrides".to_string()),
8127                workspace: test_workspace(),
8128                workspace_id: None,
8129                root_id: None,
8130                provider: None,
8131                model: None,
8132                selection_mode: None,
8133                tool_allowlist,
8134                developer_instructions,
8135                external_tools,
8136                runner: None,
8137            })
8138            .await
8139            .unwrap()
8140            .thread_id;
8141        let mut rx = runtime.subscribe_events();
8142        let turn_id = runtime
8143            .start_turn(StartTurnRequest {
8144                thread_id,
8145                message: "hello".to_string(),
8146                images: Vec::new(),
8147                provider_override: None,
8148                model_override: None,
8149                reasoning_override: None,
8150                workspace: test_workspace(),
8151                instructions: crate::default_instructions(),
8152                developer_context: None,
8153                task_ledger_required: false,
8154            })
8155            .await
8156            .unwrap();
8157        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8158            loop {
8159                let envelope = rx.recv().await.unwrap();
8160                if envelope.turn_id.as_deref() != Some(&turn_id) {
8161                    continue;
8162                }
8163                match envelope.event {
8164                    RoderEvent::TurnCompleted(_) => break,
8165                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8166                    _ => {}
8167                }
8168            }
8169        })
8170        .await
8171        .unwrap();
8172        requests.lock().unwrap().pop().expect("captured request")
8173    }
8174
8175    #[tokio::test]
8176    async fn thread_tool_allowlist_filters_only_that_thread() {
8177        let requests = Arc::new(StdMutex::new(Vec::new()));
8178        let thread_root =
8179            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8180        let mut builder = ExtensionRegistryBuilder::new();
8181        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8182            requests: requests.clone(),
8183        }));
8184        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8185            base_path: thread_root.clone(),
8186        }));
8187        builder.tool_contributor(Arc::new(ProfileToolContributor));
8188        let runtime = Arc::new(
8189            Runtime::new(
8190                builder.build().unwrap(),
8191                RuntimeConfig {
8192                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8193                    default_model: "gpt-5.5".to_string(),
8194                    model_profiles: std::collections::HashMap::from([(
8195                        "gpt-5.5".to_string(),
8196                        test_model_profile("gpt-5.5"),
8197                    )]),
8198                    ..RuntimeConfig::default()
8199                },
8200            )
8201            .unwrap(),
8202        );
8203
8204        let allowlisted = captured_thread_override_request(
8205            &runtime,
8206            &requests,
8207            vec!["edit".to_string()],
8208            None,
8209            Vec::new(),
8210        )
8211        .await;
8212        let unrestricted =
8213            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8214                .await;
8215
8216        let allowlisted_names = allowlisted
8217            .tools
8218            .iter()
8219            .map(|tool| tool.name.as_str())
8220            .collect::<Vec<_>>();
8221        assert_eq!(allowlisted_names, vec!["edit"]);
8222        let unrestricted_names = unrestricted
8223            .tools
8224            .iter()
8225            .map(|tool| tool.name.as_str())
8226            .collect::<Vec<_>>();
8227        assert!(unrestricted_names.contains(&"edit"));
8228        assert!(unrestricted_names.len() > 1);
8229
8230        let _ = std::fs::remove_dir_all(thread_root);
8231    }
8232
8233    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8234    fn runtime_with_edit_allowlist(
8235        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8236        thread_root: &std::path::Path,
8237    ) -> Arc<Runtime> {
8238        let mut builder = ExtensionRegistryBuilder::new();
8239        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8240            requests: requests.clone(),
8241        }));
8242        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8243            base_path: thread_root.to_path_buf(),
8244        }));
8245        builder.tool_contributor(Arc::new(ProfileToolContributor));
8246        Arc::new(
8247            Runtime::new(
8248                builder.build().unwrap(),
8249                RuntimeConfig {
8250                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8251                    default_model: "gpt-5.5".to_string(),
8252                    tool_allowlist: vec!["edit".to_string()],
8253                    model_profiles: std::collections::HashMap::from([(
8254                        "gpt-5.5".to_string(),
8255                        test_model_profile("gpt-5.5"),
8256                    )]),
8257                    ..RuntimeConfig::default()
8258                },
8259            )
8260            .unwrap(),
8261        )
8262    }
8263
8264    #[tokio::test]
8265    async fn runtime_and_thread_allowlists_intersect() {
8266        let requests = Arc::new(StdMutex::new(Vec::new()));
8267        let thread_root = std::env::temp_dir().join(format!(
8268            "roder-allowlist-intersect-{}",
8269            uuid::Uuid::new_v4()
8270        ));
8271        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8272
8273        let request = captured_thread_override_request(
8274            &runtime,
8275            &requests,
8276            vec!["edit".to_string(), "write_file".to_string()],
8277            None,
8278            Vec::new(),
8279        )
8280        .await;
8281
8282        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8283        let names = request
8284            .tools
8285            .iter()
8286            .map(|tool| tool.name.as_str())
8287            .collect::<Vec<_>>();
8288        assert_eq!(names, vec!["edit"]);
8289
8290        let _ = std::fs::remove_dir_all(thread_root);
8291    }
8292
8293    #[tokio::test]
8294    async fn route_tool_call_denies_tools_outside_allowlists() {
8295        let requests = Arc::new(StdMutex::new(Vec::new()));
8296        let thread_root =
8297            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8298        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8299        let thread_id = runtime
8300            .create_thread_with(CreateThreadRequest {
8301                title: Some("Dispatch allowlist".to_string()),
8302                workspace: test_workspace(),
8303                workspace_id: None,
8304                root_id: None,
8305                provider: None,
8306                model: None,
8307                selection_mode: None,
8308                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8309                developer_instructions: None,
8310                external_tools: Vec::new(),
8311                runner: None,
8312            })
8313            .await
8314            .unwrap()
8315            .thread_id;
8316
8317        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8318        let result = runtime
8319            .route_tool_call(
8320                &thread_id,
8321                &"turn-allowlist-dispatch".to_string(),
8322                roder_api::inference::ToolCallCompleted {
8323                    id: "call-1".to_string(),
8324                    name: "write_file".to_string(),
8325                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8326                },
8327                None,
8328                None,
8329            )
8330            .await
8331            .unwrap();
8332
8333        assert!(result.is_error);
8334        assert!(
8335            result
8336                .result
8337                .contains("not permitted by the tool allowlist"),
8338            "unexpected result: {}",
8339            result.result
8340        );
8341
8342        let _ = std::fs::remove_dir_all(thread_root);
8343    }
8344
8345    #[tokio::test]
8346    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8347        let requests = Arc::new(StdMutex::new(Vec::new()));
8348        let thread_root =
8349            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8350        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8351
8352        // A response that mixes agent_swarm with another tool is denied wholesale:
8353        // both calls get an error result with retry guidance, and no tool runs.
8354        let results = runtime
8355            .route_tool_calls(
8356                &"thread-swarm".to_string(),
8357                &"turn-swarm".to_string(),
8358                vec![
8359                    roder_api::inference::ToolCallCompleted {
8360                        id: "swarm-1".to_string(),
8361                        name: "agent_swarm".to_string(),
8362                        arguments: "{}".to_string(),
8363                    },
8364                    roder_api::inference::ToolCallCompleted {
8365                        id: "read-1".to_string(),
8366                        name: "read_file".to_string(),
8367                        arguments: "{}".to_string(),
8368                    },
8369                ],
8370                true,
8371                None,
8372                None,
8373            )
8374            .await
8375            .unwrap();
8376
8377        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8378        assert_eq!(results[0].id, "swarm-1");
8379        assert_eq!(results[1].id, "read-1");
8380        for result in &results {
8381            assert!(result.is_error);
8382            assert!(
8383                result.result.contains("only tool call"),
8384                "unexpected result: {}",
8385                result.result
8386            );
8387        }
8388
8389        let _ = std::fs::remove_dir_all(thread_root);
8390    }
8391
8392    #[tokio::test]
8393    async fn route_tool_calls_emits_agent_swarm_started_event() {
8394        let requests = Arc::new(StdMutex::new(Vec::new()));
8395        let thread_root =
8396            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8397        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8398        let mut events = runtime.subscribe_events();
8399
8400        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8401        // the event bus before dispatch, with the child count parsed from args.
8402        let _ = runtime
8403            .route_tool_calls(
8404                &"thread-swarm".to_string(),
8405                &"turn-swarm".to_string(),
8406                vec![roder_api::inference::ToolCallCompleted {
8407                    id: "swarm-1".to_string(),
8408                    name: "agent_swarm".to_string(),
8409                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8410                        .to_string(),
8411                }],
8412                true,
8413                None,
8414                None,
8415            )
8416            .await
8417            .unwrap();
8418
8419        let mut started_child_count = None;
8420        for _ in 0..16 {
8421            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8422                .await
8423                .unwrap()
8424                .unwrap();
8425            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8426                started_child_count = Some(event.child_count);
8427                assert_eq!(event.tool_id, "swarm-1");
8428                break;
8429            }
8430        }
8431        assert_eq!(started_child_count, Some(2));
8432
8433        let _ = std::fs::remove_dir_all(thread_root);
8434    }
8435
8436    #[test]
8437    fn agent_swarm_child_count_sums_items_and_resumes() {
8438        assert_eq!(
8439            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8440            3
8441        );
8442        assert_eq!(
8443            agent_swarm_child_count(
8444                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8445            ),
8446            2
8447        );
8448        // Malformed input is lenient.
8449        assert_eq!(agent_swarm_child_count("not json"), 0);
8450    }
8451
8452    #[test]
8453    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8454        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8455        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8456        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8457        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8458        // Not a swarm result.
8459        assert_eq!(parse_swarm_counts("just text"), None);
8460    }
8461
8462    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8463    struct SignalledFailureEngine {
8464        started: tokio::sync::mpsc::UnboundedSender<()>,
8465        proceed: Arc<tokio::sync::Notify>,
8466    }
8467
8468    #[async_trait::async_trait]
8469    impl InferenceEngine for SignalledFailureEngine {
8470        fn id(&self) -> String {
8471            roder_api::catalog::PROVIDER_MOCK.to_string()
8472        }
8473
8474        fn capabilities(&self) -> InferenceCapabilities {
8475            InferenceCapabilities::coding_agent_default()
8476        }
8477
8478        async fn list_models(
8479            &self,
8480            _ctx: InferenceProviderContext<'_>,
8481        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8482            Ok(roder_api::catalog::models_for_provider(
8483                roder_api::catalog::PROVIDER_MOCK,
8484                true,
8485            ))
8486        }
8487
8488        async fn stream_turn(
8489            &self,
8490            _ctx: InferenceTurnContext<'_>,
8491            _request: AgentInferenceRequest,
8492        ) -> anyhow::Result<InferenceEventStream> {
8493            let _ = self.started.send(());
8494            self.proceed.notified().await;
8495            anyhow::bail!("engine failed mid-turn")
8496        }
8497    }
8498
8499    #[tokio::test]
8500    async fn failed_turn_sweeps_pending_external_tool_calls() {
8501        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8502        let proceed = Arc::new(tokio::sync::Notify::new());
8503        let mut builder = ExtensionRegistryBuilder::new();
8504        builder.inference_engine(Arc::new(SignalledFailureEngine {
8505            started: started_tx,
8506            proceed: proceed.clone(),
8507        }));
8508        let runtime =
8509            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8510        let mut rx = runtime.subscribe_events();
8511        let turn_id = runtime
8512            .start_turn(StartTurnRequest {
8513                thread_id: "thread-sweep".to_string(),
8514                message: "go".to_string(),
8515                images: Vec::new(),
8516                provider_override: None,
8517                model_override: None,
8518                reasoning_override: None,
8519                workspace: test_workspace(),
8520                instructions: InstructionBundle {
8521                    system: None,
8522                    developer: None,
8523                    developer_context: None,
8524                },
8525                developer_context: None,
8526                task_ledger_required: false,
8527            })
8528            .await
8529            .unwrap();
8530        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8531            .await
8532            .unwrap()
8533            .unwrap();
8534
8535        let (tx, _pending_rx) = oneshot::channel();
8536        runtime.pending_external_tool_calls.lock().await.insert(
8537            "exttool-sweep-test".to_string(),
8538            PendingExternalToolCall {
8539                thread_id: "thread-sweep".to_string(),
8540                turn_id: turn_id.clone(),
8541                tool_id: "call-1".to_string(),
8542                tool_name: "acme_lookup".to_string(),
8543                tx,
8544            },
8545        );
8546        proceed.notify_one();
8547
8548        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8549            loop {
8550                let envelope = rx.recv().await.unwrap();
8551                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8552                    && event.request_id == "exttool-sweep-test"
8553                {
8554                    break event.outcome;
8555                }
8556            }
8557        })
8558        .await
8559        .expect("turn failure must resolve pending external tool calls");
8560        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8561        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8562    }
8563
8564    #[tokio::test]
8565    async fn thread_developer_instructions_layer_under_harness_prompt() {
8566        let requests = Arc::new(StdMutex::new(Vec::new()));
8567        let thread_root = std::env::temp_dir().join(format!(
8568            "roder-thread-instructions-{}",
8569            uuid::Uuid::new_v4()
8570        ));
8571        let mut builder = ExtensionRegistryBuilder::new();
8572        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8573            requests: requests.clone(),
8574        }));
8575        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8576            base_path: thread_root.clone(),
8577        }));
8578        builder.tool_contributor(Arc::new(ProfileToolContributor));
8579        let runtime =
8580            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8581
8582        let request = captured_thread_override_request(
8583            &runtime,
8584            &requests,
8585            Vec::new(),
8586            Some("You are embedded in a host app.".to_string()),
8587            Vec::new(),
8588        )
8589        .await;
8590
8591        let system = request.instructions.system.expect("system instructions");
8592        assert!(system.starts_with("You are Roder"));
8593        let developer = request
8594            .instructions
8595            .developer
8596            .expect("developer instructions");
8597        assert!(developer.starts_with("You are embedded in a host app."));
8598
8599        let plain =
8600            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8601                .await;
8602        assert_eq!(plain.instructions.developer, None);
8603
8604        let _ = std::fs::remove_dir_all(thread_root);
8605    }
8606
8607    #[tokio::test]
8608    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8609        let requests = Arc::new(StdMutex::new(Vec::new()));
8610        let thread_root = std::env::temp_dir().join(format!(
8611            "roder-thread-external-tools-{}",
8612            uuid::Uuid::new_v4()
8613        ));
8614        let mut builder = ExtensionRegistryBuilder::new();
8615        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8616            requests: requests.clone(),
8617        }));
8618        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8619            base_path: thread_root.clone(),
8620        }));
8621        builder.tool_contributor(Arc::new(ProfileToolContributor));
8622        let runtime =
8623            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8624
8625        let external_tools = vec![
8626            ToolSpec {
8627                name: "acme_lookup".to_string(),
8628                description: "Look up Acme workspace state.".to_string(),
8629                parameters: serde_json::json!({
8630                    "type": "object",
8631                    "properties": { "query": { "type": "string" } },
8632                    "required": ["query"]
8633                }),
8634            },
8635            ToolSpec {
8636                name: "edit".to_string(),
8637                description: "Host-managed edit.".to_string(),
8638                parameters: serde_json::json!({ "type": "object" }),
8639            },
8640        ];
8641        let request =
8642            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8643                .await;
8644
8645        let acme = request
8646            .tools
8647            .iter()
8648            .find(|tool| tool.name == "acme_lookup")
8649            .expect("external tool advertised");
8650        assert_eq!(acme.description, "Look up Acme workspace state.");
8651        assert_eq!(acme.parameters["required"][0], "query");
8652        let edits = request
8653            .tools
8654            .iter()
8655            .filter(|tool| tool.name == "edit")
8656            .collect::<Vec<_>>();
8657        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8658        assert_eq!(edits[0].description, "Host-managed edit.");
8659
8660        let plain =
8661            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8662                .await;
8663        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8664        let plain_edit = plain
8665            .tools
8666            .iter()
8667            .find(|tool| tool.name == "edit")
8668            .expect("builtin edit advertised on plain thread");
8669        assert_eq!(plain_edit.description, "edit test tool");
8670
8671        let _ = std::fs::remove_dir_all(thread_root);
8672    }
8673
8674    #[tokio::test]
8675    async fn model_switch_injects_summary_and_records_profile_segments() {
8676        let requests = Arc::new(StdMutex::new(Vec::new()));
8677        let thread_root = std::env::temp_dir().join(format!(
8678            "roder-model-switch-thread-{}",
8679            uuid::Uuid::new_v4()
8680        ));
8681        let mut builder = ExtensionRegistryBuilder::new();
8682        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8683            requests: requests.clone(),
8684        }));
8685        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8686            base_path: thread_root.clone(),
8687        }));
8688        builder.tool_contributor(Arc::new(ProfileToolContributor));
8689        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8690        claude_profile.provider_family = ProviderFamily::Anthropic;
8691        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8692        let runtime = Arc::new(
8693            Runtime::new(
8694                builder.build().unwrap(),
8695                RuntimeConfig {
8696                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8697                    default_model: "gpt-5.5".to_string(),
8698                    model_profiles: std::collections::HashMap::from([
8699                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8700                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8701                    ]),
8702                    ..RuntimeConfig::default()
8703                },
8704            )
8705            .unwrap(),
8706        );
8707        let thread_id = runtime
8708            .create_thread_with(CreateThreadRequest {
8709                title: Some("Model switch".to_string()),
8710                workspace: test_workspace(),
8711                workspace_id: None,
8712                root_id: None,
8713                provider: None,
8714                model: None,
8715                selection_mode: None,
8716                tool_allowlist: Vec::new(),
8717                developer_instructions: None,
8718                external_tools: Vec::new(),
8719                runner: None,
8720            })
8721            .await
8722            .unwrap()
8723            .thread_id;
8724        let mut rx = runtime.subscribe_events();
8725        for (message, model_override) in [
8726            ("first turn", None),
8727            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8728        ] {
8729            let turn_id = runtime
8730                .start_turn(StartTurnRequest {
8731                    thread_id: thread_id.clone(),
8732                    message: message.to_string(),
8733                    images: Vec::new(),
8734                    provider_override: None,
8735                    model_override,
8736                    reasoning_override: None,
8737                    workspace: test_workspace(),
8738                    instructions: InstructionBundle::default(),
8739                    developer_context: None,
8740                    task_ledger_required: false,
8741                })
8742                .await
8743                .unwrap();
8744            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8745                loop {
8746                    let envelope = rx.recv().await.unwrap();
8747                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8748                        continue;
8749                    }
8750                    match envelope.event {
8751                        RoderEvent::TurnCompleted(_) => break,
8752                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8753                        _ => {}
8754                    }
8755                }
8756            })
8757            .await
8758            .unwrap();
8759        }
8760
8761        let captured = requests.lock().unwrap().clone();
8762        assert_eq!(captured.len(), 2);
8763        assert!(captured[1].transcript.iter().any(|item| {
8764            matches!(
8765                item,
8766                TranscriptItem::UserMessage(message)
8767                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
8768                        && message.text.contains("previous profile mock/gpt-5.5")
8769                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
8770                        && message.text.contains("Available tools now:")
8771            )
8772        }));
8773
8774        let snapshot = runtime
8775            .thread_store
8776            .as_ref()
8777            .unwrap()
8778            .load_thread(&thread_id)
8779            .await
8780            .unwrap()
8781            .unwrap();
8782        let trace_segments = snapshot
8783            .turns
8784            .iter()
8785            .flat_map(|turn| &turn.items)
8786            .filter(|item| {
8787                matches!(
8788                    item,
8789                    TranscriptItem::ProviderMetadata(value)
8790                        if value.get("kind").and_then(serde_json::Value::as_str)
8791                            == Some(MODEL_PROFILE_TRACE_KIND)
8792                            && value.get("segment").and_then(serde_json::Value::as_str)
8793                                == Some("assistant")
8794                )
8795            })
8796            .count();
8797        assert!(trace_segments >= 2);
8798        let _ = std::fs::remove_dir_all(thread_root);
8799    }
8800
8801    struct CountingTaskTool {
8802        calls: Arc<StdMutex<u32>>,
8803    }
8804
8805    #[async_trait::async_trait]
8806    impl ToolExecutor for CountingTaskTool {
8807        fn spec(&self) -> ToolSpec {
8808            ToolSpec {
8809                name: "task".to_string(),
8810                description: "Dispatch a test subagent.".to_string(),
8811                parameters: serde_json::json!({
8812                    "type": "object",
8813                    "properties": {
8814                        "description": { "type": "string" },
8815                        "prompt": { "type": "string" },
8816                        "parent_deadline_seconds": { "type": "integer" }
8817                    },
8818                    "required": ["description", "prompt"],
8819                    "additionalProperties": false
8820                }),
8821            }
8822        }
8823
8824        async fn execute(
8825            &self,
8826            _ctx: ToolExecutionContext,
8827            call: ToolCall,
8828        ) -> anyhow::Result<ToolResult> {
8829            *self.calls.lock().unwrap() += 1;
8830            Ok(ToolResult {
8831                id: call.id,
8832                name: call.name,
8833                text: "started child".to_string(),
8834                data: serde_json::json!({}),
8835                is_error: false,
8836            })
8837        }
8838    }
8839
8840    #[tokio::test]
8841    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
8842        let captured = Arc::new(StdMutex::new(None));
8843        let mut builder = ExtensionRegistryBuilder::new();
8844        builder.inference_engine(Arc::new(CapturingEngine {
8845            request: captured.clone(),
8846        }));
8847        let runtime = Arc::new(
8848            Runtime::new(
8849                builder.build().unwrap(),
8850                RuntimeConfig {
8851                    runtime_profile: RuntimeProfile::NonInteractive,
8852                    ..RuntimeConfig::default()
8853                },
8854            )
8855            .unwrap(),
8856        );
8857        let mut rx = runtime.subscribe_events();
8858        let turn_id = runtime
8859            .start_turn(StartTurnRequest {
8860                thread_id: "thread-profile".to_string(),
8861                message: "work unattended".to_string(),
8862                images: Vec::new(),
8863                provider_override: None,
8864                model_override: None,
8865                reasoning_override: None,
8866                workspace: test_workspace(),
8867                instructions: InstructionBundle {
8868                    system: None,
8869                    developer: Some("base developer".to_string()),
8870                    developer_context: None,
8871                },
8872                developer_context: None,
8873                task_ledger_required: false,
8874            })
8875            .await
8876            .unwrap();
8877
8878        let mut observed_profile = None;
8879        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8880            loop {
8881                let envelope = rx.recv().await.unwrap();
8882                if envelope.turn_id.as_deref() != Some(&turn_id) {
8883                    continue;
8884                }
8885                match envelope.event {
8886                    RoderEvent::TurnStarted(event) => {
8887                        observed_profile = Some(event.runtime_profile);
8888                    }
8889                    RoderEvent::TurnCompleted(_) => break,
8890                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8891                    _ => {}
8892                }
8893            }
8894        })
8895        .await
8896        .unwrap();
8897
8898        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
8899        let request = captured.lock().unwrap().clone().unwrap();
8900        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
8901        let developer = request.instructions.developer.unwrap();
8902        assert!(developer.contains("base developer"));
8903        assert!(developer.contains("non-interactive profile"));
8904    }
8905
8906    #[tokio::test]
8907    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
8908        let workspace = runtime_test_workspace("global-policy-mode");
8909        let thread_root = workspace.join("threads");
8910        let mut builder = ExtensionRegistryBuilder::new();
8911        builder.inference_engine(Arc::new(FakeInferenceEngine));
8912        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8913            base_path: thread_root.clone(),
8914        }));
8915        let runtime = Runtime::new(
8916            builder.build().unwrap(),
8917            RuntimeConfig {
8918                workspace: Some(workspace.display().to_string()),
8919                ..Default::default()
8920            },
8921        )
8922        .unwrap();
8923
8924        runtime
8925            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
8926            .await
8927            .unwrap();
8928
8929        assert!(!thread_root.join("runtime").exists());
8930        let _ = std::fs::remove_dir_all(workspace);
8931    }
8932
8933    #[tokio::test]
8934    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
8935        let captured = Arc::new(StdMutex::new(None));
8936        let mut builder = ExtensionRegistryBuilder::new();
8937        builder.inference_engine(Arc::new(CapturingEngine {
8938            request: captured.clone(),
8939        }));
8940        builder.tool_contributor(Arc::new(
8941            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
8942        ));
8943        let runtime = Arc::new(
8944            Runtime::new(
8945                builder.build().unwrap(),
8946                RuntimeConfig {
8947                    runtime_profile: RuntimeProfile::Eval,
8948                    policy_mode: PolicyMode::Bypass,
8949                    agent_swarm_mode: false,
8950                    ..RuntimeConfig::default()
8951                },
8952            )
8953            .unwrap(),
8954        );
8955        let mut rx = runtime.subscribe_events();
8956        let turn_id = runtime
8957            .start_turn(StartTurnRequest {
8958                thread_id: "thread-ledger".to_string(),
8959                message: "decomposed work".to_string(),
8960                images: Vec::new(),
8961                provider_override: None,
8962                model_override: None,
8963                reasoning_override: None,
8964                workspace: test_workspace(),
8965                instructions: InstructionBundle::default(),
8966                developer_context: None,
8967                task_ledger_required: true,
8968            })
8969            .await
8970            .unwrap();
8971
8972        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8973            loop {
8974                let envelope = rx.recv().await.unwrap();
8975                if envelope.turn_id.as_deref() == Some(&turn_id)
8976                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
8977                {
8978                    break;
8979                }
8980            }
8981        })
8982        .await
8983        .unwrap();
8984
8985        let request = captured.lock().unwrap().clone().unwrap();
8986        let developer = request.instructions.developer.unwrap();
8987        assert!(developer.contains("Task Ledger Required"));
8988        assert!(developer.contains("task_ledger.update"));
8989        let tool_names: Vec<_> = request
8990            .tools
8991            .iter()
8992            .map(|tool| tool.name.as_str())
8993            .collect();
8994        assert!(
8995            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
8996            "tool names: {tool_names:?}"
8997        );
8998        assert_eq!(
8999            request.tool_choice,
9000            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9001        );
9002        assert_eq!(request.tools.len(), 1);
9003        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9004    }
9005
9006    #[tokio::test]
9007    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9008        let requests = Arc::new(StdMutex::new(Vec::new()));
9009        let mut builder = ExtensionRegistryBuilder::new();
9010        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9011            calls: StdMutex::new(0),
9012            requests: requests.clone(),
9013        }));
9014        builder.tool_contributor(Arc::new(
9015            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9016        ));
9017        let runtime = Arc::new(
9018            Runtime::new(
9019                builder.build().unwrap(),
9020                RuntimeConfig {
9021                    runtime_profile: RuntimeProfile::Eval,
9022                    policy_mode: PolicyMode::Bypass,
9023                    agent_swarm_mode: false,
9024                    ..RuntimeConfig::default()
9025                },
9026            )
9027            .unwrap(),
9028        );
9029        let mut rx = runtime.subscribe_events();
9030        let turn_id = runtime
9031            .start_turn(StartTurnRequest {
9032                thread_id: "thread-ledger-completion".to_string(),
9033                message: "write the answer file".to_string(),
9034                images: Vec::new(),
9035                provider_override: None,
9036                model_override: None,
9037                reasoning_override: None,
9038                workspace: test_workspace(),
9039                instructions: InstructionBundle::default(),
9040                developer_context: None,
9041                task_ledger_required: true,
9042            })
9043            .await
9044            .unwrap();
9045
9046        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9047            loop {
9048                let envelope = rx.recv().await.unwrap();
9049                if envelope.turn_id.as_deref() != Some(&turn_id) {
9050                    continue;
9051                }
9052                match envelope.event {
9053                    RoderEvent::TurnCompleted(_) => break,
9054                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9055                    _ => {}
9056                }
9057            }
9058        })
9059        .await
9060        .unwrap();
9061
9062        let requests = requests.lock().unwrap().clone();
9063        assert_eq!(requests.len(), 4);
9064        assert!(requests[2].transcript.iter().any(|item| {
9065            matches!(
9066                item,
9067                TranscriptItem::UserMessage(message)
9068                    if message.text.contains("Task Ledger Completion Required")
9069                        && message.text.contains("Write /app/result.txt")
9070            )
9071        }));
9072        assert!(requests[3].transcript.iter().any(|item| {
9073            matches!(
9074                item,
9075                TranscriptItem::ToolResult(result)
9076                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9077                        && result.result.contains("Task ledger: 2/2 completed")
9078            )
9079        }));
9080    }
9081
9082    #[tokio::test]
9083    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9084        let requests = Arc::new(StdMutex::new(Vec::new()));
9085        let mut builder = ExtensionRegistryBuilder::new();
9086        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9087            calls: StdMutex::new(0),
9088            requests: requests.clone(),
9089        }));
9090        builder.tool_contributor(Arc::new(
9091            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9092        ));
9093        let runtime = Arc::new(
9094            Runtime::new(
9095                builder.build().unwrap(),
9096                RuntimeConfig {
9097                    runtime_profile: RuntimeProfile::Eval,
9098                    policy_mode: PolicyMode::Bypass,
9099                    agent_swarm_mode: false,
9100                    turn_deadline_seconds: Some(120),
9101                    ..RuntimeConfig::default()
9102                },
9103            )
9104            .unwrap(),
9105        );
9106        let mut rx = runtime.subscribe_events();
9107        let turn_id = runtime
9108            .start_turn(StartTurnRequest {
9109                thread_id: "thread-ledger-checkpoint".to_string(),
9110                message: "write the answer file".to_string(),
9111                images: Vec::new(),
9112                provider_override: None,
9113                model_override: None,
9114                reasoning_override: None,
9115                workspace: test_workspace(),
9116                instructions: InstructionBundle::default(),
9117                developer_context: None,
9118                task_ledger_required: true,
9119            })
9120            .await
9121            .unwrap();
9122
9123        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9124            loop {
9125                let envelope = rx.recv().await.unwrap();
9126                if envelope.turn_id.as_deref() != Some(&turn_id) {
9127                    continue;
9128                }
9129                match envelope.event {
9130                    RoderEvent::TurnCompleted(_) => break,
9131                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9132                    _ => {}
9133                }
9134            }
9135        })
9136        .await
9137        .unwrap();
9138
9139        let requests = requests.lock().unwrap().clone();
9140        assert!(requests.len() >= 2);
9141        assert!(requests[1].transcript.iter().any(|item| {
9142            matches!(
9143                item,
9144                TranscriptItem::UserMessage(message)
9145                    if message.text.contains("Scoreable Output Checkpoint")
9146                        && message.text.contains("ensure the required output file(s) exist")
9147                        && message.text.contains("Write /app/result.txt")
9148            )
9149        }));
9150    }
9151
9152    #[test]
9153    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9154        let prompt = task_ledger_deadline_completion_prompt(
9155            12,
9156            30,
9157            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9158        );
9159
9160        assert!(prompt.contains("12 seconds remain"));
9161        assert!(prompt.contains("create or update the required scoreable output files"));
9162        assert!(prompt.contains("write /app/result.txt"));
9163        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9164    }
9165
9166    #[test]
9167    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9168        let prompt = task_ledger_scoreable_checkpoint_prompt(
9169            120,
9170            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9171        );
9172
9173        assert!(prompt.contains("120 seconds remain"));
9174        assert!(prompt.contains("best evidence-backed answer"));
9175        assert!(prompt.contains("even if provisional"));
9176        assert!(prompt.contains("preserve that candidate"));
9177        assert!(prompt.contains("partial-coverage"));
9178        assert!(prompt.contains("write /app/result.txt"));
9179        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9180    }
9181
9182    #[test]
9183    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9184        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9185        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9186            id: "ledger-open".to_string(),
9187            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9188            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9189                .to_string(),
9190            display_payload: None,
9191            is_error: false,
9192        })];
9193
9194        let (_, action) = inference_timeout_deadline(
9195            deadline,
9196            RuntimeProfile::Eval,
9197            true,
9198            30,
9199            false,
9200            0,
9201            &transcript,
9202        )
9203        .unwrap();
9204
9205        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9206    }
9207
9208    #[tokio::test]
9209    async fn verification_gate_forces_eval_code_changes_through_review() {
9210        let mut builder = ExtensionRegistryBuilder::new();
9211        builder.inference_engine(Arc::new(VerificationGateEngine {
9212            calls: StdMutex::new(0),
9213        }));
9214        builder.tool_contributor(Arc::new(WriteFileContributor));
9215        builder.tool_contributor(Arc::new(
9216            roder_ext_verification::VerificationToolContributor,
9217        ));
9218        let runtime = Arc::new(
9219            Runtime::new(
9220                builder.build().unwrap(),
9221                RuntimeConfig {
9222                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9223                    default_model: "mock".to_string(),
9224                    runtime_profile: RuntimeProfile::Eval,
9225                    policy_mode: PolicyMode::Bypass,
9226                    agent_swarm_mode: false,
9227                    ..RuntimeConfig::default()
9228                },
9229            )
9230            .unwrap(),
9231        );
9232        let mut rx = runtime.subscribe_events();
9233        let turn_id = runtime
9234            .start_turn(StartTurnRequest {
9235                thread_id: "thread-verification".to_string(),
9236                message: "write code".to_string(),
9237                images: Vec::new(),
9238                provider_override: None,
9239                model_override: None,
9240                reasoning_override: None,
9241                workspace: test_workspace(),
9242                instructions: InstructionBundle::default(),
9243                developer_context: None,
9244                task_ledger_required: false,
9245            })
9246            .await
9247            .unwrap();
9248
9249        let mut saw_required = false;
9250        let mut saw_completed = false;
9251        let mut final_text = String::new();
9252        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9253            loop {
9254                let envelope = rx.recv().await.unwrap();
9255                if envelope.turn_id.as_deref() != Some(&turn_id) {
9256                    continue;
9257                }
9258                match envelope.event {
9259                    RoderEvent::VerificationRequired(event) => {
9260                        saw_required = true;
9261                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9262                    }
9263                    RoderEvent::VerificationCompleted(event) => {
9264                        saw_completed = true;
9265                        assert!(event.passed);
9266                    }
9267                    RoderEvent::InferenceEventReceived(event) => {
9268                        if let InferenceEvent::MessageDelta(delta) = event.event {
9269                            final_text.push_str(&delta.text);
9270                        }
9271                    }
9272                    RoderEvent::TurnCompleted(_) => break,
9273                    _ => {}
9274                }
9275            }
9276        })
9277        .await
9278        .unwrap();
9279
9280        assert!(saw_required);
9281        assert!(saw_completed);
9282        assert!(final_text.contains("verified final"));
9283    }
9284
9285    #[tokio::test]
9286    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9287        let requests = Arc::new(StdMutex::new(Vec::new()));
9288        let mut builder = ExtensionRegistryBuilder::new();
9289        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9290            calls: StdMutex::new(0),
9291            requests: requests.clone(),
9292        }));
9293        builder.tool_contributor(Arc::new(WriteFileContributor));
9294        builder.tool_contributor(Arc::new(
9295            roder_ext_verification::VerificationToolContributor,
9296        ));
9297        let runtime = Arc::new(
9298            Runtime::new(
9299                builder.build().unwrap(),
9300                RuntimeConfig {
9301                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9302                    default_model: "gpt-5.5".to_string(),
9303                    runtime_profile: RuntimeProfile::Eval,
9304                    policy_mode: PolicyMode::Bypass,
9305                    agent_swarm_mode: false,
9306                    ..RuntimeConfig::default()
9307                },
9308            )
9309            .unwrap(),
9310        );
9311        let mut rx = runtime.subscribe_events();
9312        let turn_id = runtime
9313            .start_turn(StartTurnRequest {
9314                thread_id: "thread-speed-policy".to_string(),
9315                message: "write code".to_string(),
9316                images: Vec::new(),
9317                provider_override: None,
9318                model_override: None,
9319                reasoning_override: None,
9320                workspace: test_workspace(),
9321                instructions: InstructionBundle::default(),
9322                developer_context: None,
9323                task_ledger_required: false,
9324            })
9325            .await
9326            .unwrap();
9327
9328        let mut saw_speed_policy_event = false;
9329        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9330            loop {
9331                let envelope = rx.recv().await.unwrap();
9332                if envelope.turn_id.as_deref() != Some(&turn_id) {
9333                    continue;
9334                }
9335                match envelope.event {
9336                    RoderEvent::InferenceStarted(event) => {
9337                        if event.speed_policy.is_some() {
9338                            saw_speed_policy_event = true;
9339                        }
9340                    }
9341                    RoderEvent::TurnCompleted(_) => break,
9342                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9343                    _ => {}
9344                }
9345            }
9346        })
9347        .await
9348        .unwrap();
9349
9350        let requests = requests.lock().unwrap().clone();
9351        assert!(saw_speed_policy_event);
9352        assert!(requests.len() >= 4);
9353        assert!(requests.iter().all(|request| {
9354            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9355                && request.model.model == "gpt-5.5"
9356        }));
9357        assert_eq!(
9358            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9359            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9360        );
9361        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9362        assert_eq!(
9363            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9364            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9365        );
9366        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9367        assert_eq!(
9368            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9369            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9370        );
9371        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9372        assert_eq!(
9373            requests[2]
9374                .metadata
9375                .pointer("/speedPolicy/phase")
9376                .and_then(serde_json::Value::as_str),
9377            Some("verification")
9378        );
9379    }
9380
9381    #[tokio::test]
9382    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9383        let mut builder = ExtensionRegistryBuilder::new();
9384        builder.inference_engine(Arc::new(DeadlineEngine));
9385        let runtime = Arc::new(
9386            Runtime::new(
9387                builder.build().unwrap(),
9388                RuntimeConfig {
9389                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9390                    default_model: "mock".to_string(),
9391                    runtime_profile: RuntimeProfile::Eval,
9392                    turn_deadline_seconds: Some(1),
9393                    ..RuntimeConfig::default()
9394                },
9395            )
9396            .unwrap(),
9397        );
9398        let mut rx = runtime.subscribe_events();
9399        let turn_id = runtime
9400            .start_turn(StartTurnRequest {
9401                thread_id: "thread-deadline".to_string(),
9402                message: "slow work".to_string(),
9403                images: Vec::new(),
9404                provider_override: None,
9405                model_override: None,
9406                reasoning_override: None,
9407                workspace: test_workspace(),
9408                instructions: InstructionBundle::default(),
9409                developer_context: None,
9410                task_ledger_required: false,
9411            })
9412            .await
9413            .unwrap();
9414
9415        let mut saw_partial = false;
9416        let mut saw_deadline = false;
9417        let mut failed_kind = None;
9418        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9419            loop {
9420                let envelope = rx.recv().await.unwrap();
9421                if envelope.turn_id.as_deref() != Some(&turn_id) {
9422                    continue;
9423                }
9424                match envelope.event {
9425                    RoderEvent::TurnPartialResult(event) => {
9426                        saw_partial = event.summary.contains("partial turn state");
9427                    }
9428                    RoderEvent::TurnDeadlineExceeded(event) => {
9429                        saw_deadline = event.partial_result.contains("transcript items");
9430                    }
9431                    RoderEvent::TurnFailed(event) => {
9432                        failed_kind = event.error_kind;
9433                        break;
9434                    }
9435                    _ => {}
9436                }
9437            }
9438        })
9439        .await
9440        .unwrap();
9441
9442        for _ in 0..20 {
9443            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9444                break;
9445            }
9446            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9447        }
9448        assert!(saw_partial);
9449        assert!(saw_deadline);
9450        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9451        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9452    }
9453
9454    #[tokio::test]
9455    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9456        let calls = Arc::new(StdMutex::new(0));
9457        let mut builder = ExtensionRegistryBuilder::new();
9458        builder.inference_engine(Arc::new(CapturingEngine {
9459            request: Arc::new(StdMutex::new(None)),
9460        }));
9461        let task_tool = Arc::new(CountingTaskTool {
9462            calls: calls.clone(),
9463        });
9464        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9465        let runtime = Arc::new(
9466            Runtime::new(
9467                builder.build().unwrap(),
9468                RuntimeConfig {
9469                    policy_mode: PolicyMode::Bypass,
9470                    agent_swarm_mode: false,
9471                    ..RuntimeConfig::default()
9472                },
9473            )
9474            .unwrap(),
9475        );
9476
9477        let result = runtime
9478            .route_tool_call(
9479                &"thread-deadline-task".to_string(),
9480                &"turn-deadline-task".to_string(),
9481                ToolCallCompleted {
9482                    id: "task-1".to_string(),
9483                    name: "task".to_string(),
9484                    arguments: serde_json::json!({
9485                        "description": "inspect",
9486                        "prompt": "read"
9487                    })
9488                    .to_string(),
9489                },
9490                None,
9491                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9492            )
9493            .await
9494            .unwrap();
9495
9496        assert!(result.is_error);
9497        assert!(result.result.contains("deadline policy skipped"));
9498        assert_eq!(*calls.lock().unwrap(), 0);
9499    }
9500
9501    struct TestToolContributor {
9502        tool: Arc<dyn ToolExecutor>,
9503    }
9504
9505    impl ToolContributor for TestToolContributor {
9506        fn id(&self) -> String {
9507            "test-tool".to_string()
9508        }
9509
9510        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9511            registry.register(self.tool.clone())
9512        }
9513    }
9514
9515    #[tokio::test]
9516    async fn agent_swarm_mode_override_is_per_thread() {
9517        let runtime = Runtime::fake().unwrap();
9518        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9519
9520        // Off everywhere by default.
9521        assert!(
9522            !runtime
9523                .effective_agent_swarm_mode_for_thread("thread-a")
9524                .await
9525        );
9526        assert!(
9527            !runtime
9528                .effective_agent_swarm_mode_for_thread("thread-b")
9529                .await
9530        );
9531
9532        // Enabling on thread-a does not leak into thread-b.
9533        assert!(
9534            runtime
9535                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9536                .await
9537        );
9538        assert!(
9539            runtime
9540                .effective_agent_swarm_mode_for_thread("thread-a")
9541                .await
9542        );
9543        assert!(
9544            !runtime
9545                .effective_agent_swarm_mode_for_thread("thread-b")
9546                .await
9547        );
9548
9549        // A per-thread `off` override wins over a runtime-global `on` default.
9550        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9551        runtime
9552            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9553            .await;
9554        assert!(
9555            !runtime
9556                .effective_agent_swarm_mode_for_thread("thread-b")
9557                .await,
9558            "explicit per-thread off overrides the global on default"
9559        );
9560        // A thread with no override still follows the global default.
9561        assert!(
9562            runtime
9563                .effective_agent_swarm_mode_for_thread("thread-c")
9564                .await,
9565            "threads without an override follow the runtime-global default"
9566        );
9567    }
9568
9569    #[tokio::test]
9570    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9571        let runtime = Runtime::fake().unwrap();
9572        let mut events = runtime.subscribe_events();
9573        runtime
9574            .set_agent_swarm_mode_for_thread(
9575                "thread-xyz",
9576                true,
9577                roder_api::subagents::AgentSwarmModeTrigger::Task,
9578            )
9579            .await;
9580        let mut saw = false;
9581        for _ in 0..8 {
9582            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9583                .await
9584                .unwrap()
9585                .unwrap();
9586            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9587                assert_eq!(event.thread_id, "thread-xyz");
9588                assert!(event.enabled);
9589                assert_eq!(
9590                    event.trigger,
9591                    roder_api::subagents::AgentSwarmModeTrigger::Task
9592                );
9593                saw = true;
9594                break;
9595            }
9596        }
9597        assert!(
9598            saw,
9599            "expected an AgentSwarmModeChanged event for the thread"
9600        );
9601    }
9602}